From bd0da97d5a3c6baac126e09522259577a7794ab5 Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:41:20 +0000 Subject: [PATCH 01/11] test: reproduce admission, pin and cache invariants --- test/helpers/effects.ts | 56 +++++++++++++++++++ test/invariants/i2-no-loss.test.ts | 22 ++++++++ test/invariants/i5-admission-parity.test.ts | 34 +++++++++++ .../invariants/i7-effects-containment.test.ts | 20 +++++++ 4 files changed, 132 insertions(+) create mode 100644 test/helpers/effects.ts create mode 100644 test/invariants/i2-no-loss.test.ts create mode 100644 test/invariants/i5-admission-parity.test.ts create mode 100644 test/invariants/i7-effects-containment.test.ts diff --git a/test/helpers/effects.ts b/test/helpers/effects.ts new file mode 100644 index 00000000..131764b4 --- /dev/null +++ b/test/helpers/effects.ts @@ -0,0 +1,56 @@ +import { lstat, readdir, readFile, readlink } from "node:fs/promises"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; + +export interface EffectSnapshot { + tree: Record; + claims: Record; + git?: { index: string; refs: string; head: string }; +} +export interface EffectPolicy { + // Exact relative paths only, each with a fixture-specific reason. No globs or prefixes. + exclude?: Record; + manifests?: string[]; + git?: boolean; +} + +export async function snapshot(root: string, policy: EffectPolicy = {}): Promise { + for (const [path, reason] of Object.entries(policy.exclude ?? {})) { + if (!reason.trim() || path.startsWith("/") || path.split("/").some(p => p === ".." || p === "." || !p) + || /[*?\[\]]/.test(path)) throw new Error("Exclusions require an exact relative path and reason"); + } + const result: EffectSnapshot = { tree: {}, claims: {} }; + async function visit(relative: string): Promise { + const path = join(root, relative); + const stat = await lstat(path); + if (stat.isDirectory()) { + if (relative) result.tree[relative] = { kind: "directory", mode: stat.mode & 0o7777 }; + for (const name of (await readdir(path)).sort()) await visit(relative ? `${relative}/${name}` : name); + } else if (!Object.hasOwn(policy.exclude ?? {}, relative)) { + result.tree[relative] = stat.isSymbolicLink() + ? { kind: "symlink", mode: stat.mode & 0o7777, target: await readlink(path) } + : { kind: "file", mode: stat.mode & 0o7777, bytes: (await readFile(path)).toString("base64") }; + } else if ((policy.manifests ?? []).includes(relative)) { + throw new Error("Manifest claims cannot be excluded"); + } + } + await visit(""); + for (const path of policy.manifests ?? []) result.claims[path] = JSON.parse(await readFile(join(root, path), "utf8")); + if (policy.git) { + const git = (...args: string[]) => execFileSync("git", ["-C", root, ...args], { encoding: "utf8" }); + result.git = { index: git("ls-files", "--stage", "-z"), refs: git("show-ref", "--head"), head: git("symbolic-ref", "-q", "HEAD") }; + } + return result; +} + +export function diffEffects(before: EffectSnapshot, after: EffectSnapshot): string[] { + const changes: string[] = []; + for (const section of ["tree", "claims", "git"] as const) { + const left = before[section] ?? {}; + const right = after[section] ?? {}; + for (const key of [...new Set([...Object.keys(left), ...Object.keys(right)])].sort()) { + if (JSON.stringify(left[key as keyof typeof left]) !== JSON.stringify(right[key as keyof typeof right])) changes.push(`${section}:${key}`); + } + } + return changes; +} diff --git a/test/invariants/i2-no-loss.test.ts b/test/invariants/i2-no-loss.test.ts new file mode 100644 index 00000000..918772aa --- /dev/null +++ b/test/invariants/i2-no-loss.test.ts @@ -0,0 +1,22 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { GitSourceDriver } from "../../src/source/git.js"; +import { diffEffects, snapshot } from "../helpers/effects.js"; + +it("read_missing_git_preserves_cache", async () => { + const root = await mkdtemp(join(tmpdir(), "invariant-i2-")); + try { + const driver = new GitSourceDriver(); + // Local nonexistent source: clone must fail without network access. + const resolved = await driver.resolve(`git+file://${join(root, "missing.git")}`, { cacheRoot: join(root, "cache") }); + await mkdir(resolved.resolvedPath, { recursive: true }); + await writeFile(join(resolved.resolvedPath, "sentinel"), "preserve me"); + const before = await snapshot(root); + const failure = await driver.fetch(resolved).then(() => undefined, error => error); + expect(failure).toBeInstanceOf(Error); + // Expected: a read refuses an incomplete checkout without deleting existing bytes. + expect(diffEffects(before, await snapshot(root))).toEqual([]); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts new file mode 100644 index 00000000..78bad990 --- /dev/null +++ b/test/invariants/i5-admission-parity.test.ts @@ -0,0 +1,34 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { createCombinedInstallPlan, applyCombinedInstallPlan, readInstallManifest, type DesiredArtifact } from "../../src/install/index.js"; +import { targetMappingSchema, type AdapterConfig } from "../../src/model/adapter.js"; +import { hashPath } from "../../src/utils/fs.js"; +import { diffEffects, snapshot } from "../helpers/effects.js"; + +it("stale_merge_metadata_rejected_in_all_paths", async () => { + const root = await mkdtemp(join(tmpdir(), "invariant-i5-")); + try { + const sourcePath = join(root, "source.json"); + await writeFile(sourcePath, JSON.stringify({ managed: { value: "current" } })); + const adapter: AdapterConfig = { name: "synthetic", targets: { settings: { + local: targetMappingSchema.parse({ enabled: true, dest: "config.json", merge: "json-deep" }), + } } }; + const artifact: DesiredArtifact = { type: "settings", name: "synthetic", sourcePath, stagedPath: sourcePath, + relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed" }; + await applyCombinedInstallPlan(await createCombinedInstallPlan([artifact], adapter, root)); + const manifest = (await readInstallManifest(root, adapter.name))!; + manifest.entries[0]!.mergeRemoval = { managed: { value: "stale" } }; + // Source hash is unchanged; user-owned fields must survive both admission paths. + await writeFile(join(root, "config.json"), JSON.stringify({ managed: { value: "current" }, user: { keep: true } })); + const before = await snapshot(root); + const preview = await createCombinedInstallPlan([artifact], adapter, root, manifest); + expect(diffEffects(before, await snapshot(root))).toEqual([]); + const failure = await applyCombinedInstallPlan(preview).then(() => undefined, error => error); + expect(failure).toBeInstanceOf(Error); + expect(String(failure)).toMatch(/managed|contribution|merge/i); + // Expected: preview blocks the same stale contribution that apply rejects. + expect(preview.hasBlockingChanges).toBe(true); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/test/invariants/i7-effects-containment.test.ts b/test/invariants/i7-effects-containment.test.ts new file mode 100644 index 00000000..927827f4 --- /dev/null +++ b/test/invariants/i7-effects-containment.test.ts @@ -0,0 +1,20 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { refuseNewPinCollisions } from "../../src/lifecycle/pin-preflight.js"; +import { diffEffects, snapshot } from "../helpers/effects.js"; + +it("pin_provenance_and_feature_floor_enforced", async () => { + const root = await mkdtemp(join(tmpdir(), "invariant-i7-")); + try { + const before = await snapshot(root); + const warnings: string[] = []; + const failure = await refuseNewPinCollisions(root, { + packages: [{ name: "synthetic-pin", source: join(root, "unresolvable"), mode: "pinned" }], + }, message => warnings.push(message)).then(() => undefined, error => error); + expect(diffEffects(before, await snapshot(root))).toEqual([]); + // Expected: an unresolved approved pin refuses; warning and success are insufficient. + expect({ refused: failure instanceof Error, warnings }).toMatchObject({ refused: true }); + } finally { await rm(root, { recursive: true, force: true }); } +}); From 156ee7e3a7f685ac6fa906ffc88d45fb0f7d635f Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:42:14 +0000 Subject: [PATCH 02/11] test: use supported local git source syntax --- test/invariants/i2-no-loss.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/invariants/i2-no-loss.test.ts b/test/invariants/i2-no-loss.test.ts index 918772aa..2b295b41 100644 --- a/test/invariants/i2-no-loss.test.ts +++ b/test/invariants/i2-no-loss.test.ts @@ -10,7 +10,7 @@ it("read_missing_git_preserves_cache", async () => { try { const driver = new GitSourceDriver(); // Local nonexistent source: clone must fail without network access. - const resolved = await driver.resolve(`git+file://${join(root, "missing.git")}`, { cacheRoot: join(root, "cache") }); + const resolved = await driver.resolve(`git:file://${join(root, "missing.git")}`, { cacheRoot: join(root, "cache") }); await mkdir(resolved.resolvedPath, { recursive: true }); await writeFile(join(resolved.resolvedPath, "sentinel"), "preserve me"); const before = await snapshot(root); From 38ccbc060164d72b1eaf51cd914ac8eee7de6b5f Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:43:13 +0000 Subject: [PATCH 03/11] test: complete invariant fixture inputs --- test/invariants/i5-admission-parity.test.ts | 2 +- test/invariants/i7-effects-containment.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts index 78bad990..48011504 100644 --- a/test/invariants/i5-admission-parity.test.ts +++ b/test/invariants/i5-admission-parity.test.ts @@ -16,7 +16,7 @@ it("stale_merge_metadata_rejected_in_all_paths", async () => { local: targetMappingSchema.parse({ enabled: true, dest: "config.json", merge: "json-deep" }), } } }; const artifact: DesiredArtifact = { type: "settings", name: "synthetic", sourcePath, stagedPath: sourcePath, - relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed" }; + relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed", meta: {} }; await applyCombinedInstallPlan(await createCombinedInstallPlan([artifact], adapter, root)); const manifest = (await readInstallManifest(root, adapter.name))!; manifest.entries[0]!.mergeRemoval = { managed: { value: "stale" } }; diff --git a/test/invariants/i7-effects-containment.test.ts b/test/invariants/i7-effects-containment.test.ts index 927827f4..142d7d02 100644 --- a/test/invariants/i7-effects-containment.test.ts +++ b/test/invariants/i7-effects-containment.test.ts @@ -11,7 +11,7 @@ it("pin_provenance_and_feature_floor_enforced", async () => { const before = await snapshot(root); const warnings: string[] = []; const failure = await refuseNewPinCollisions(root, { - packages: [{ name: "synthetic-pin", source: join(root, "unresolvable"), mode: "pinned" }], + schemaVersion: 1, packages: [{ name: "synthetic-pin", source: join(root, "unresolvable"), mode: "pinned" }], }, message => warnings.push(message)).then(() => undefined, error => error); expect(diffEffects(before, await snapshot(root))).toEqual([]); // Expected: an unresolved approved pin refuses; warning and success are insufficient. From cca0024d3f2caacf7169436c2d0ec4a52887ca74 Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:44:41 +0000 Subject: [PATCH 04/11] test: inventory Commander commands and reconcile runner test identities --- scripts/test-command-map.json | 23 ++++ scripts/test-command-matrix.mjs | 130 ++++++++++++++++++++ scripts/test-command-reporter.mjs | 28 +++++ test/invariants/i5-admission-parity.test.ts | 2 +- 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 scripts/test-command-map.json create mode 100644 scripts/test-command-matrix.mjs create mode 100644 scripts/test-command-reporter.mjs diff --git a/scripts/test-command-map.json b/scripts/test-command-map.json new file mode 100644 index 00000000..8e093074 --- /dev/null +++ b/scripts/test-command-map.json @@ -0,0 +1,23 @@ +{ + "schemaVersion": 1, + "links": [ + { + "file": "test/cli-verb-redesign.test.ts", + "name": "CLI verb redesign > previews and applies Git cache pruning", + "commands": ["cache prune"], + "evidence": "Runs cache prune in preview and apply modes and checks retained cache entries." + }, + { + "file": "test/cli-verb-redesign.test.ts", + "name": "CLI verb redesign > drafts a registry publish submission from a GitHub URL without writing workspace state", + "commands": ["registry publish"], + "evidence": "Invokes registry publish and checks its draft and workspace effects." + }, + { + "file": "test/cli-verb-redesign.test.ts", + "name": "CLI verb redesign > doctor suggests the Copilot user companion skill install without writing files", + "commands": ["doctor"], + "evidence": "Invokes doctor and checks the suggested install and unchanged tree." + } + ] +} diff --git a/scripts/test-command-matrix.mjs b/scripts/test-command-matrix.mjs new file mode 100644 index 00000000..2bdd99e9 --- /dev/null +++ b/scripts/test-command-matrix.mjs @@ -0,0 +1,130 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile, mkdtemp, rm, readdir } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { pathToFileURL } from "node:url"; +import { spawnSync, execFileSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { build } from "esbuild"; + +const root = process.cwd(); +const require = createRequire(import.meta.url); +const compare = (a, b) => a < b ? -1 : a > b ? 1 : 0; +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +const read = path => readFile(join(root, path), "utf8"); + +export function counts(tests) { + const pass = tests.filter(t => t.status === "passed").length; + const fail = tests.filter(t => t.status === "failed").length; + return { registered: tests.length, executed: pass + fail, pass, fail, + skipped: tests.filter(t => t.status === "skipped").length, + pending: tests.filter(t => t.status === "pending").length }; +} + +export function reconcile(commands, registered, results, links) { + const byId = new Map(registered.map(t => [t.id, t])); + const resultIds = new Set(results.map(t => t.id)); + if (byId.size !== registered.length || resultIds.size !== results.length + || byId.size !== resultIds.size || [...byId.keys()].some(id => !resultIds.has(id))) { + throw new Error("Registered and executed runner IDs do not reconcile"); + } + const attributed = new Map(commands.map(command => [command, new Set()])); + const assigned = new Set(); + const selectors = new Set(); + for (const link of links) { + const key = `${link.file}::${link.name}`; + if (selectors.has(key) || !link.evidence?.trim() || !link.commands.length) throw new Error(`Invalid mapping: ${key}`); + selectors.add(key); + const matches = registered.filter(t => t.file === link.file && t.name === link.name); + if (matches.length !== 1) throw new Error(`Mapping must match exactly one registered test: ${key} (${matches.length})`); + for (const command of link.commands) { + if (!attributed.has(command)) throw new Error(`Unknown parser command: ${command}`); + attributed.get(command).add(matches[0].id); + assigned.add(matches[0].id); + } + } + const rows = commands.map(command => { + const tests = results.filter(t => attributed.get(command).has(t.id)).sort((a,b) => compare(a.id,b.id)); + return { command, ...counts(tests), tests }; + }); + return { rows, uniqueAttributed: assigned.size, + unattributed: results.filter(t => !assigned.has(t.id)).sort((a,b) => compare(a.id,b.id)) }; +} + +async function parserCommands(scratch) { + const source = await read("src/cli/index.ts"); + const marker = "try {\n await main();"; + if (source.split(marker).length !== 2) throw new Error("CLI entry boundary changed; inspect inventory loader"); + // Load the actual Commander tree, replacing only the executable entrypoint. + const code = `${source.slice(0, source.indexOf(marker))}\nexport { program };\n`; + const output = join(scratch, "parser.mjs"); + await build({ stdin: { contents: code, resolveDir: join(root,"src/cli"), sourcefile: "index.ts", loader: "ts" }, + bundle: true, packages: "external", platform: "node", format: "esm", outfile: output, write: true }); + const { program } = await import(pathToFileURL(output)); + const commands = [program.name()]; + function visit(parent, prefix) { + for (const command of parent.commands) { + const path = [...prefix, command.name()]; + commands.push(path.join(" ")); + visit(command, path); + } + } + visit(program, []); + if (new Set(commands).size !== commands.length) throw new Error("Duplicate parser command path"); + return commands.sort(compare); +} + +async function sourceReceipt() { + // Hash exact source/test/script/config bytes; no absolute paths, timestamps or self-referential output. + const files = []; + async function visit(dir) { + for (const entry of (await readdir(join(root,dir), { withFileTypes: true })).sort((a,b) => compare(a.name,b.name))) { + const path = `${dir}/${entry.name}`; + if (entry.isDirectory()) await visit(path); + else if (entry.isFile() && !path.includes("/.command-matrix-")) files.push([path, hash(await readFile(join(root,path)))]); + } + } + for (const dir of ["src", "test", "scripts"]) await visit(dir); + for (const path of ["package.json", "pnpm-lock.yaml", "vitest.config.ts", "tsconfig.json"]) files.push([path,hash(await read(path))]); + return { inputDigest: hash(JSON.stringify(files.sort((a,b) => compare(a[0],b[0])))), + parserSha256: hash(await read("src/cli/index.ts")), + vitest: require("vitest/package.json").version, + node: process.versions.node }; +} + +async function main() { + // Inside the repository so external imports resolve the installed dependencies. + const scratch = await mkdtemp(join(root, "scripts/.command-matrix-")); + const reportRoot = await mkdtemp(join(tmpdir(), "command-matrix-")); + try { + const commands = await parserCommands(scratch); + const report = join(reportRoot,"runner.json"); + const json = join(reportRoot,"vitest.json"); + const runner = spawnSync(process.execPath, [require.resolve("vitest/vitest.mjs"), "run", + "--maxWorkers=2", "--reporter=json", "--reporter=./scripts/test-command-reporter.mjs", `--outputFile=${json}`], { + cwd: root, env: { ...process.env, COMMAND_MATRIX_REPORT: report }, encoding: "utf8", maxBuffer: 40 * 1024 * 1024, + }); + if (runner.error) throw runner.error; + const observed = JSON.parse(await readFile(report,"utf8")); + const official = JSON.parse(await readFile(json,"utf8")); + const officialTests = official.testResults.flatMap(file => file.assertionResults); + if (officialTests.length !== observed.tests.length) throw new Error("JSON reporter test count mismatch"); + const mapping = JSON.parse(await read("scripts/test-command-map.json")); + if (mapping.schemaVersion !== 1) throw new Error("Unsupported mapping schema"); + const matrix = reconcile(commands, observed.registered, observed.tests, mapping.links); + const output = { schemaVersion: 1, receipt: await sourceReceipt(), + commandCount: commands.length, zeroTestCommands: matrix.rows.filter(row => !row.registered).map(row => row.command), + totals: counts(observed.tests), uniqueAttributed: matrix.uniqueAttributed, + runnerExitCode: runner.status, moduleErrors: observed.moduleErrors, unhandledErrors: observed.unhandledErrors, + commands: matrix.rows, unattributed: matrix.unattributed }; + const destination = resolve(process.argv[2] ?? "docs/test-command-matrix.json"); + await writeFile(destination, `${JSON.stringify(output,null,2)}\n`); + console.log(JSON.stringify({ commandCount: output.commandCount, zeroTestCommands: output.zeroTestCommands.length, + ...output.totals, uniqueAttributed: output.uniqueAttributed, output: relative(root,destination) })); + if (runner.status !== 0 || observed.moduleErrors.length || observed.unhandledErrors) process.exitCode = 1; + } finally { + await rm(scratch, { recursive: true, force: true }); + await rm(reportRoot, { recursive: true, force: true }); + } +} +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) await main(); diff --git a/scripts/test-command-reporter.mjs b/scripts/test-command-reporter.mjs new file mode 100644 index 00000000..632e4ffd --- /dev/null +++ b/scripts/test-command-reporter.mjs @@ -0,0 +1,28 @@ +import { writeFileSync } from "node:fs"; + +// Uses runner-owned IDs, including parameter expansion, rather than source-text guesses. +export default class CommandMatrixReporter { + registered = new Map(); + onTestModuleCollected(module) { + for (const test of module.children.allTests()) { + if (this.registered.has(test.id)) throw new Error(`Duplicate runner test ID: ${test.id}`); + this.registered.set(test.id, { + id: test.id, file: module.relativeModuleId, name: test.fullName, + expectedFailure: test.options.fails === true, + }); + } + } + onTestRunEnd(modules, errors, reason) { + const tests = []; + for (const module of modules) for (const test of module.children.allTests()) { + const registered = this.registered.get(test.id); + if (!registered) throw new Error(`Unregistered runner test: ${test.id}`); + tests.push({ ...registered, status: test.result().state }); + } + writeFileSync(process.env.COMMAND_MATRIX_REPORT, JSON.stringify({ + registered: [...this.registered.values()], tests, reason, + moduleErrors: modules.filter(module => !module.ok()).map(module => module.relativeModuleId), + unhandledErrors: errors.length, + })); + } +} diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts index 48011504..55c6c235 100644 --- a/test/invariants/i5-admission-parity.test.ts +++ b/test/invariants/i5-admission-parity.test.ts @@ -16,7 +16,7 @@ it("stale_merge_metadata_rejected_in_all_paths", async () => { local: targetMappingSchema.parse({ enabled: true, dest: "config.json", merge: "json-deep" }), } } }; const artifact: DesiredArtifact = { type: "settings", name: "synthetic", sourcePath, stagedPath: sourcePath, - relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed", meta: {} }; + relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed", meta: { dependencyRole: "root" } }; await applyCombinedInstallPlan(await createCombinedInstallPlan([artifact], adapter, root)); const manifest = (await readInstallManifest(root, adapter.name))!; manifest.entries[0]!.mergeRemoval = { managed: { value: "stale" } }; From 50f452001c2e3e5f9f6125e25ef4f7388a87888d Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:46:14 +0000 Subject: [PATCH 05/11] test: verify effect snapshots and command attribution accounting --- docs/testing-invariants.md | 21 +++++++++++++ scripts/test-command-matrix.mjs | 14 +++------ test/command-matrix.test.ts | 17 ++++++++++ test/effects-harness.test.ts | 35 +++++++++++++++++++++ test/invariants/i5-admission-parity.test.ts | 2 +- 5 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 docs/testing-invariants.md create mode 100644 test/command-matrix.test.ts create mode 100644 test/effects-harness.test.ts diff --git a/docs/testing-invariants.md b/docs/testing-invariants.md new file mode 100644 index 00000000..464ac1df --- /dev/null +++ b/docs/testing-invariants.md @@ -0,0 +1,21 @@ +# Invariant baseline and command attribution + +Run `node scripts/test-command-matrix.mjs [output.json]` from the repository root after installing dependencies. The script runs the entire Vitest suite; failures remain in the output and yield a nonzero exit. It loads the actual Commander tree with only its executable entrypoint removed, including root, groups and hidden commands. No command handler runs during inventory. + +`scripts/test-command-map.json` is deliberately conservative: every link names exactly one runner-registered file/full-name pair and documents why that test covers the command. Imports and shared files do not imply command coverage. Add audited links as coverage is established; an unattributed test is not evidence of absent behavior. Unknown commands, missing or ambiguous cases, duplicate IDs and registration/result mismatches refuse generation. Parameterized cases remain individual runner IDs. Per-command counts overlap; only the union of IDs is a suite total. IDs are Vitest IDs, stable for the same runner/project/module/test order. The separate JSON reporter reconciles the overall count. + +The report records input-byte and parser hashes, runner and Node versions. It excludes timing, absolute paths and stack traces. Unattributed tests are explicitly listed to reconcile the full runner inventory. `it.fails` outcomes count as runner passes and carry `expectedFailure: true`; this is not proof that the invariant holds. Skipped and pending outcomes are separate from executed cases. Generated reports are observations of one environment and are not committed as a portable status baseline. + +## Effects harness + +`test/helpers/effects.ts` records file bytes in base64, permission bits, directories and symlink targets without following links. Explicit manifest paths record complete parsed claims as well as original bytes. With `git: true`, the snapshot also records the staged index, refs and symbolic HEAD; the filesystem tree includes raw Git metadata. Locks under the fixture root are included. No timestamps are recorded. There are no default path exclusions. A caller may exclude only an exact file or symlink path with a nonempty reason; directories are always traversed and manifest claims cannot be excluded. Compare snapshots with `diffEffects`; unknown changed paths remain visible. This is an isolated filesystem observer, not a syscall/network/process tracer. + +## Expected failures + +The invariant tests use synthetic fixtures and no live targets. Each assertion is first reproduced as an ordinary failing test, then marked `it.fails`. A product fix must remove that marker in the same change: an unexpected pass fails CI. Do not replace the marker with skip or todo. + +| Invariant | Case | Observed defect | +| --- | --- | --- | +| I2 | `read_missing_git_preserves_cache` | Fetch removes an existing incomplete cache directory and its sentinel before a failing local clone. | +| I5 | `stale_merge_metadata_rejected_in_all_paths` | Preview accepts unchanged source with stale contribution metadata while apply rejects the contribution. | +| I7 | `pin_provenance_and_feature_floor_enforced` | Unresolvable local pin emits a warning and returns successfully. This case covers unreachable pins only. | diff --git a/scripts/test-command-matrix.mjs b/scripts/test-command-matrix.mjs index 2bdd99e9..afcb39ed 100644 --- a/scripts/test-command-matrix.mjs +++ b/scripts/test-command-matrix.mjs @@ -76,16 +76,10 @@ async function parserCommands(scratch) { async function sourceReceipt() { // Hash exact source/test/script/config bytes; no absolute paths, timestamps or self-referential output. - const files = []; - async function visit(dir) { - for (const entry of (await readdir(join(root,dir), { withFileTypes: true })).sort((a,b) => compare(a.name,b.name))) { - const path = `${dir}/${entry.name}`; - if (entry.isDirectory()) await visit(path); - else if (entry.isFile() && !path.includes("/.command-matrix-")) files.push([path, hash(await readFile(join(root,path)))]); - } - } - for (const dir of ["src", "test", "scripts"]) await visit(dir); - for (const path of ["package.json", "pnpm-lock.yaml", "vitest.config.ts", "tsconfig.json"]) files.push([path,hash(await read(path))]); + const tracked = execFileSync("git", ["ls-files", "-z"], { cwd: root, encoding: "utf8" }).split("\0").filter(Boolean); + const paths = tracked.filter(path => /^(src|test|scripts)\//.test(path) + || ["package.json", "pnpm-lock.yaml", "vitest.config.ts", "tsconfig.json"].includes(path)); + const files = await Promise.all(paths.sort(compare).map(async path => [path, hash(await readFile(join(root,path)))])); return { inputDigest: hash(JSON.stringify(files.sort((a,b) => compare(a[0],b[0])))), parserSha256: hash(await read("src/cli/index.ts")), vitest: require("vitest/package.json").version, diff --git a/test/command-matrix.test.ts b/test/command-matrix.test.ts new file mode 100644 index 00000000..99b9d9b8 --- /dev/null +++ b/test/command-matrix.test.ts @@ -0,0 +1,17 @@ +import { expect, it } from "vitest"; +// @ts-expect-error The CLI support script is native ESM JavaScript. +import { reconcile, counts } from "../scripts/test-command-matrix.mjs"; + +it("reconciles shared command attribution without double-counting unique runner IDs", () => { + const registered = [{ id: "a", file: "case.ts", name: "parameter 1" }, { id: "b", file: "case.ts", name: "parameter 2" }]; + const results = registered.map(test => ({ ...test, status: "passed" })); + const matrix = reconcile(["one", "two", "zero"], registered, results, + [{ file: "case.ts", name: "parameter 1", commands: ["one", "two"], evidence: "shared fixture" }]); + expect(matrix.uniqueAttributed).toBe(1); + expect(matrix.rows.map((row: { registered: number }) => row.registered)).toEqual([1,1,0]); + expect(matrix.unattributed.map((test: { id: string }) => test.id)).toEqual(["b"]); + expect(() => reconcile(["one"], registered, [results[0]], [])).toThrow(/reconcile/); + expect(() => reconcile(["one"], registered, results, [{ file: "case.ts", name: "missing", commands: ["one"], evidence: "fixture" }])).toThrow(/exactly one/); + expect(() => reconcile(["one"], registered, results, [{ file: "case.ts", name: "parameter 1", commands: ["unknown"], evidence: "fixture" }])).toThrow(/Unknown/); + expect(counts([{ status: "passed" }, { status: "failed" }, { status: "skipped" }, { status: "pending" }])).toEqual({ registered: 4, executed: 2, pass: 1, fail: 1, skipped: 1, pending: 1 }); +}); diff --git a/test/effects-harness.test.ts b/test/effects-harness.test.ts new file mode 100644 index 00000000..d8f8582a --- /dev/null +++ b/test/effects-harness.test.ts @@ -0,0 +1,35 @@ +import { execFileSync } from "node:child_process"; +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { diffEffects, snapshot } from "./helpers/effects.js"; + +it("detects bytes, modes, symlinks, claims, index and refs without following links", async () => { + const root = await mkdtemp(join(tmpdir(), "effects-harness-")); + try { + const git = (...args: string[]) => execFileSync("git", ["-C", root, ...args]); + git("init", "-b", "main"); + git("config", "user.name", "Synthetic"); + git("config", "user.email", "synthetic@example.invalid"); + await writeFile(join(root, "file"), Buffer.from([0,255,1])); + await writeFile(join(root, "manifest.json"), JSON.stringify({ entries: [{ owner: "alpha" }] })); + await symlink("file", join(root,"link")); + git("add", "file"); git("commit", "-m", "fixture"); + const policy = { manifests: ["manifest.json"], git: true }; + const before = await snapshot(root, policy); + expect(diffEffects(before, await snapshot(root,policy))).toEqual([]); + await writeFile(join(root,"file"), Buffer.from([0,255,2])); + await chmod(join(root,"file"), 0o755); + await rm(join(root,"link")); await symlink("missing",join(root,"link")); + await writeFile(join(root,"manifest.json"), JSON.stringify({ entries: [{ owner: "beta" }] })); + git("add", "file"); git("branch", "other"); + const changes = diffEffects(before, await snapshot(root,policy)); + expect(changes).toEqual(expect.arrayContaining(["tree:file", "tree:link", "claims:manifest.json", "git:index", "git:refs"])); + await expect(snapshot(root, { exclude: { "**": "invalid" } })).rejects.toThrow(/exact/); + await expect(snapshot(root, { exclude: { "file": "" } })).rejects.toThrow(/reason/); + const excluded = { exclude: { "file": "This fixture explicitly ignores the mutable file payload." } }; + expect((await snapshot(root,excluded)).tree.file).toBeUndefined(); + await expect(snapshot(root, { manifests: ["manifest.json"], exclude: { "manifest.json": "invalid claim exclusion" } })).rejects.toThrow(/claims/); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts index 55c6c235..51762835 100644 --- a/test/invariants/i5-admission-parity.test.ts +++ b/test/invariants/i5-admission-parity.test.ts @@ -16,7 +16,7 @@ it("stale_merge_metadata_rejected_in_all_paths", async () => { local: targetMappingSchema.parse({ enabled: true, dest: "config.json", merge: "json-deep" }), } } }; const artifact: DesiredArtifact = { type: "settings", name: "synthetic", sourcePath, stagedPath: sourcePath, - relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed", meta: { dependencyRole: "root" } }; + relativePath: "source.json", kind: "file", hash: await hashPath(sourcePath), channel: "managed", meta: { dependencyRole: "root", owners: ["synthetic"], logicalSelector: "settings/synthetic" } }; await applyCombinedInstallPlan(await createCombinedInstallPlan([artifact], adapter, root)); const manifest = (await readInstallManifest(root, adapter.name))!; manifest.entries[0]!.mergeRemoval = { managed: { value: "stale" } }; From 6eb5abc1d95f70a56091ed55bc20699836b2b86b Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:47:24 +0000 Subject: [PATCH 06/11] test: retain verified invariant failures and publish CI command matrix --- .github/workflows/ci.yml | 9 ++++++++- scripts/test-command-matrix.mjs | 4 +++- test/invariants/i2-no-loss.test.ts | 2 +- test/invariants/i5-admission-parity.test.ts | 2 +- test/invariants/i7-effects-containment.test.ts | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a40ca332..703d09ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,4 +70,11 @@ jobs: - name: Test env: SYNCWHEEL_SOURCE_ROOT: ${{ github.workspace }}/syncwheel - run: pnpm test + run: node scripts/test-command-matrix.mjs "${{ runner.temp }}/test-command-matrix.json" + + - name: Upload command test matrix + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-command-matrix + path: ${{ runner.temp }}/test-command-matrix.json diff --git a/scripts/test-command-matrix.mjs b/scripts/test-command-matrix.mjs index afcb39ed..a31569b8 100644 --- a/scripts/test-command-matrix.mjs +++ b/scripts/test-command-matrix.mjs @@ -5,10 +5,12 @@ import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; import { spawnSync, execFileSync } from "node:child_process"; import { createRequire } from "node:module"; -import { build } from "esbuild"; + const root = process.cwd(); const require = createRequire(import.meta.url); +// Use the bundler shipped with the pinned, direct tsup development dependency. +const { build } = createRequire(require.resolve("tsup/package.json"))("esbuild"); const compare = (a, b) => a < b ? -1 : a > b ? 1 : 0; const hash = bytes => createHash("sha256").update(bytes).digest("hex"); const read = path => readFile(join(root, path), "utf8"); diff --git a/test/invariants/i2-no-loss.test.ts b/test/invariants/i2-no-loss.test.ts index 2b295b41..7f579679 100644 --- a/test/invariants/i2-no-loss.test.ts +++ b/test/invariants/i2-no-loss.test.ts @@ -5,7 +5,7 @@ import { expect, it } from "vitest"; import { GitSourceDriver } from "../../src/source/git.js"; import { diffEffects, snapshot } from "../helpers/effects.js"; -it("read_missing_git_preserves_cache", async () => { +it.fails("read_missing_git_preserves_cache", async () => { const root = await mkdtemp(join(tmpdir(), "invariant-i2-")); try { const driver = new GitSourceDriver(); diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts index 51762835..37c38922 100644 --- a/test/invariants/i5-admission-parity.test.ts +++ b/test/invariants/i5-admission-parity.test.ts @@ -7,7 +7,7 @@ import { targetMappingSchema, type AdapterConfig } from "../../src/model/adapter import { hashPath } from "../../src/utils/fs.js"; import { diffEffects, snapshot } from "../helpers/effects.js"; -it("stale_merge_metadata_rejected_in_all_paths", async () => { +it.fails("stale_merge_metadata_rejected_in_all_paths", async () => { const root = await mkdtemp(join(tmpdir(), "invariant-i5-")); try { const sourcePath = join(root, "source.json"); diff --git a/test/invariants/i7-effects-containment.test.ts b/test/invariants/i7-effects-containment.test.ts index 142d7d02..7a9919fb 100644 --- a/test/invariants/i7-effects-containment.test.ts +++ b/test/invariants/i7-effects-containment.test.ts @@ -5,7 +5,7 @@ import { expect, it } from "vitest"; import { refuseNewPinCollisions } from "../../src/lifecycle/pin-preflight.js"; import { diffEffects, snapshot } from "../helpers/effects.js"; -it("pin_provenance_and_feature_floor_enforced", async () => { +it.fails("pin_provenance_and_feature_floor_enforced", async () => { const root = await mkdtemp(join(tmpdir(), "invariant-i7-")); try { const before = await snapshot(root); From f31d029a681ebad165e18a092550ce107e1bf1a1 Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:49:05 +0000 Subject: [PATCH 07/11] test: resolve runner executable through its exported package metadata --- scripts/test-command-matrix.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-command-matrix.mjs b/scripts/test-command-matrix.mjs index a31569b8..1e173d27 100644 --- a/scripts/test-command-matrix.mjs +++ b/scripts/test-command-matrix.mjs @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { readFile, writeFile, mkdtemp, rm, readdir } from "node:fs/promises"; -import { join, relative, resolve } from "node:path"; +import { dirname, join, relative, resolve } from "node:path"; import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; import { spawnSync, execFileSync } from "node:child_process"; @@ -96,7 +96,7 @@ async function main() { const commands = await parserCommands(scratch); const report = join(reportRoot,"runner.json"); const json = join(reportRoot,"vitest.json"); - const runner = spawnSync(process.execPath, [require.resolve("vitest/vitest.mjs"), "run", + const runner = spawnSync(process.execPath, [join(dirname(require.resolve("vitest/package.json")), "vitest.mjs"), "run", "--maxWorkers=2", "--reporter=json", "--reporter=./scripts/test-command-reporter.mjs", `--outputFile=${json}`], { cwd: root, env: { ...process.env, COMMAND_MATRIX_REPORT: report }, encoding: "utf8", maxBuffer: 40 * 1024 * 1024, }); From e7ddfecc1319cb1151d9d4d510e16244775cec7c Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:50:25 +0000 Subject: [PATCH 08/11] test: require stale admission refusal before additional effects --- test/invariants/i5-admission-parity.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts index 37c38922..b3f446b7 100644 --- a/test/invariants/i5-admission-parity.test.ts +++ b/test/invariants/i5-admission-parity.test.ts @@ -27,8 +27,11 @@ it.fails("stale_merge_metadata_rejected_in_all_paths", async () => { expect(diffEffects(before, await snapshot(root))).toEqual([]); const failure = await applyCombinedInstallPlan(preview).then(() => undefined, error => error); expect(failure).toBeInstanceOf(Error); - expect(String(failure)).toMatch(/managed|contribution|merge/i); // Expected: preview blocks the same stale contribution that apply rejects. expect(preview.hasBlockingChanges).toBe(true); + expect(preview.operations).toEqual(expect.arrayContaining([expect.objectContaining({ + action: "conflict", blockedReason: expect.stringMatching(/contribution|merge/i), + })])); + expect(diffEffects(before, await snapshot(root))).toEqual([]); } finally { await rm(root, { recursive: true, force: true }); } }); From adb319dfea54504e1281396e57eaeb3e3b239a55 Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:51:49 +0000 Subject: [PATCH 09/11] test: include root permissions and raw Git index in effect snapshots --- docs/testing-invariants.md | 2 +- test/helpers/effects.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/testing-invariants.md b/docs/testing-invariants.md index 464ac1df..553e67e2 100644 --- a/docs/testing-invariants.md +++ b/docs/testing-invariants.md @@ -8,7 +8,7 @@ The report records input-byte and parser hashes, runner and Node versions. It ex ## Effects harness -`test/helpers/effects.ts` records file bytes in base64, permission bits, directories and symlink targets without following links. Explicit manifest paths record complete parsed claims as well as original bytes. With `git: true`, the snapshot also records the staged index, refs and symbolic HEAD; the filesystem tree includes raw Git metadata. Locks under the fixture root are included. No timestamps are recorded. There are no default path exclusions. A caller may exclude only an exact file or symlink path with a nonempty reason; directories are always traversed and manifest claims cannot be excluded. Compare snapshots with `diffEffects`; unknown changed paths remain visible. This is an isolated filesystem observer, not a syscall/network/process tracer. +`test/helpers/effects.ts` records file bytes in base64, permission bits, directories and symlink targets without following links. Explicit manifest paths record complete parsed claims as well as original bytes. With `git: true`, the snapshot also records the staged index (semantic entries and raw bytes), refs and symbolic HEAD; the filesystem tree includes raw Git metadata. Locks under the fixture root are included. Filesystem atime, mtime, ctime and birthtime are not recorded; all listed content and permission fields are compared exactly. There are no default path exclusions. A caller may exclude only an exact file or symlink path with a nonempty reason; directories are always traversed and manifest claims cannot be excluded. Compare snapshots with `diffEffects`; unknown changed paths remain visible. This is an isolated filesystem observer, not a syscall/network/process tracer. ## Expected failures diff --git a/test/helpers/effects.ts b/test/helpers/effects.ts index 131764b4..2140d645 100644 --- a/test/helpers/effects.ts +++ b/test/helpers/effects.ts @@ -1,11 +1,11 @@ import { lstat, readdir, readFile, readlink } from "node:fs/promises"; -import { join } from "node:path"; +import { isAbsolute, join, resolve } from "node:path"; import { execFileSync } from "node:child_process"; export interface EffectSnapshot { tree: Record; claims: Record; - git?: { index: string; refs: string; head: string }; + git?: { index: string; refs: string; head: string; indexBytes: string }; } export interface EffectPolicy { // Exact relative paths only, each with a fixture-specific reason. No globs or prefixes. @@ -24,7 +24,7 @@ export async function snapshot(root: string, policy: EffectPolicy = {}): Promise const path = join(root, relative); const stat = await lstat(path); if (stat.isDirectory()) { - if (relative) result.tree[relative] = { kind: "directory", mode: stat.mode & 0o7777 }; + result.tree[relative || "."] = { kind: "directory", mode: stat.mode & 0o7777 }; for (const name of (await readdir(path)).sort()) await visit(relative ? `${relative}/${name}` : name); } else if (!Object.hasOwn(policy.exclude ?? {}, relative)) { result.tree[relative] = stat.isSymbolicLink() @@ -38,7 +38,9 @@ export async function snapshot(root: string, policy: EffectPolicy = {}): Promise for (const path of policy.manifests ?? []) result.claims[path] = JSON.parse(await readFile(join(root, path), "utf8")); if (policy.git) { const git = (...args: string[]) => execFileSync("git", ["-C", root, ...args], { encoding: "utf8" }); - result.git = { index: git("ls-files", "--stage", "-z"), refs: git("show-ref", "--head"), head: git("symbolic-ref", "-q", "HEAD") }; + const indexPath = git("rev-parse", "--git-path", "index").trim(); + const indexBytes = (await readFile(isAbsolute(indexPath) ? indexPath : resolve(root,indexPath))).toString("base64"); + result.git = { indexBytes, index: git("ls-files", "--stage", "-z"), refs: git("show-ref", "--head"), head: git("symbolic-ref", "-q", "HEAD") }; } return result; } From 3429cb12619b775caf10aa61f044f561e79ce5db Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:53:02 +0000 Subject: [PATCH 10/11] test: run the declared package test command for matrix observations --- scripts/test-command-matrix.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-command-matrix.mjs b/scripts/test-command-matrix.mjs index 1e173d27..573da2e4 100644 --- a/scripts/test-command-matrix.mjs +++ b/scripts/test-command-matrix.mjs @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { readFile, writeFile, mkdtemp, rm, readdir } from "node:fs/promises"; -import { dirname, join, relative, resolve } from "node:path"; +import { join, relative, resolve } from "node:path"; import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; import { spawnSync, execFileSync } from "node:child_process"; @@ -96,7 +96,7 @@ async function main() { const commands = await parserCommands(scratch); const report = join(reportRoot,"runner.json"); const json = join(reportRoot,"vitest.json"); - const runner = spawnSync(process.execPath, [join(dirname(require.resolve("vitest/package.json")), "vitest.mjs"), "run", + const runner = spawnSync("pnpm", ["test", "--maxWorkers=2", "--reporter=json", "--reporter=./scripts/test-command-reporter.mjs", `--outputFile=${json}`], { cwd: root, env: { ...process.env, COMMAND_MATRIX_REPORT: report }, encoding: "utf8", maxBuffer: 40 * 1024 * 1024, }); From 71101a9b4f92ffad1ee817fe50842506dba202e1 Mon Sep 17 00:00:00 2001 From: Yehonal <147092+Yehonal@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:10:52 +0000 Subject: [PATCH 11/11] test: correct invariant scope and expose matrix runner failures --- docs/testing-invariants.md | 11 ++--- scripts/test-command-matrix.mjs | 37 ++++++++++------ test/command-matrix.test.ts | 43 ++++++++++++++++++- test/invariants/i2-no-loss.test.ts | 24 ++++++++--- test/invariants/i5-admission-parity.test.ts | 30 +++++++++---- .../invariants/i7-effects-containment.test.ts | 20 --------- 6 files changed, 112 insertions(+), 53 deletions(-) delete mode 100644 test/invariants/i7-effects-containment.test.ts diff --git a/docs/testing-invariants.md b/docs/testing-invariants.md index 553e67e2..e8b109dd 100644 --- a/docs/testing-invariants.md +++ b/docs/testing-invariants.md @@ -1,10 +1,10 @@ # Invariant baseline and command attribution -Run `node scripts/test-command-matrix.mjs [output.json]` from the repository root after installing dependencies. The script runs the entire Vitest suite; failures remain in the output and yield a nonzero exit. It loads the actual Commander tree with only its executable entrypoint removed, including root, groups and hidden commands. No command handler runs during inventory. +Run `node scripts/test-command-matrix.mjs [output.json]` from the repository root after installing dependencies. Without an output argument, the report is saved under the operating system temporary directory. Parser scratch files also live there. The script runs the entire Vitest suite with the default reporter streamed to stdout/stderr; failures remain in the output and yield a nonzero exit. It loads the actual Commander tree with only its executable entrypoint removed, including root, groups and hidden commands. No command handler runs during inventory. -`scripts/test-command-map.json` is deliberately conservative: every link names exactly one runner-registered file/full-name pair and documents why that test covers the command. Imports and shared files do not imply command coverage. Add audited links as coverage is established; an unattributed test is not evidence of absent behavior. Unknown commands, missing or ambiguous cases, duplicate IDs and registration/result mismatches refuse generation. Parameterized cases remain individual runner IDs. Per-command counts overlap; only the union of IDs is a suite total. IDs are Vitest IDs, stable for the same runner/project/module/test order. The separate JSON reporter reconciles the overall count. +`scripts/test-command-map.json` currently links only three commands and is deliberately conservative: every link names exactly one runner-registered file/full-name pair and documents why that test covers the command. Imports and shared files do not imply command coverage. Add audited links as coverage is established; an unattributed test is not evidence of absent behavior. Unknown commands, missing or ambiguous cases, duplicate IDs and registration/result mismatches refuse generation. Parameterized cases remain individual runner IDs. Per-command counts overlap; only the union of IDs is a suite total. IDs are Vitest IDs, stable for the same runner/project/module/test order. The separate JSON reporter reconciles the overall count. -The report records input-byte and parser hashes, runner and Node versions. It excludes timing, absolute paths and stack traces. Unattributed tests are explicitly listed to reconcile the full runner inventory. `it.fails` outcomes count as runner passes and carry `expectedFailure: true`; this is not proof that the invariant holds. Skipped and pending outcomes are separate from executed cases. Generated reports are observations of one environment and are not committed as a portable status baseline. +The report records input-byte and parser hashes, runner and Node versions. It excludes timing, absolute paths and stack traces. `unattributedCommands` lists commands with no declared test links; this is not a claim that their behavior is untested. Unattributed tests are explicitly listed to reconcile the full runner inventory. `it.fails` outcomes count as runner passes and carry `expectedFailure: true`; this is not proof that the invariant holds. Skipped and pending outcomes are separate from executed cases. Generated reports are observations of one environment and are not committed as a portable status baseline. ## Effects harness @@ -12,10 +12,11 @@ The report records input-byte and parser hashes, runner and Node versions. It ex ## Expected failures -The invariant tests use synthetic fixtures and no live targets. Each assertion is first reproduced as an ordinary failing test, then marked `it.fails`. A product fix must remove that marker in the same change: an unexpected pass fails CI. Do not replace the marker with skip or todo. +The invariant tests use synthetic fixtures and no live targets. Each expected-failure test has an adjacent ordinary test that checks the exact current defect, so a broken setup cannot masquerade as an expected failure. A product fix must remove that marker in the same change: an unexpected pass fails CI. Do not replace the marker with skip or todo. | Invariant | Case | Observed defect | | --- | --- | --- | | I2 | `read_missing_git_preserves_cache` | Fetch removes an existing incomplete cache directory and its sentinel before a failing local clone. | | I5 | `stale_merge_metadata_rejected_in_all_paths` | Preview accepts unchanged source with stale contribution metadata while apply rejects the contribution. | -| I7 | `pin_provenance_and_feature_floor_enforced` | Unresolvable local pin emits a warning and returns successfully. This case covers unreachable pins only. | + +An unreachable source during pin preflight produces a warning by design, not a defect: it does not prove a new artifact collision. The existing pin-preflight tests retain that intended behavior. diff --git a/scripts/test-command-matrix.mjs b/scripts/test-command-matrix.mjs index 573da2e4..16f21e11 100644 --- a/scripts/test-command-matrix.mjs +++ b/scripts/test-command-matrix.mjs @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { readFile, writeFile, mkdtemp, rm, readdir } from "node:fs/promises"; +import { readFile, writeFile, mkdtemp, rm } from "node:fs/promises"; import { join, relative, resolve } from "node:path"; import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; @@ -35,7 +35,7 @@ export function reconcile(commands, registered, results, links) { const selectors = new Set(); for (const link of links) { const key = `${link.file}::${link.name}`; - if (selectors.has(key) || !link.evidence?.trim() || !link.commands.length) throw new Error(`Invalid mapping: ${key}`); + if (selectors.has(key) || !link.evidence?.trim() || !Array.isArray(link.commands) || !link.commands.length) throw new Error(`Invalid mapping: ${key}`); selectors.add(key); const matches = registered.filter(t => t.file === link.file && t.name === link.name); if (matches.length !== 1) throw new Error(`Mapping must match exactly one registered test: ${key} (${matches.length})`); @@ -53,7 +53,7 @@ export function reconcile(commands, registered, results, links) { unattributed: results.filter(t => !assigned.has(t.id)).sort((a,b) => compare(a.id,b.id)) }; } -async function parserCommands(scratch) { +export async function parserCommands(scratch) { const source = await read("src/cli/index.ts"); const marker = "try {\n await main();"; if (source.split(marker).length !== 2) throw new Error("CLI entry boundary changed; inspect inventory loader"); @@ -61,7 +61,11 @@ async function parserCommands(scratch) { const code = `${source.slice(0, source.indexOf(marker))}\nexport { program };\n`; const output = join(scratch, "parser.mjs"); await build({ stdin: { contents: code, resolveDir: join(root,"src/cli"), sourcefile: "index.ts", loader: "ts" }, - bundle: true, packages: "external", platform: "node", format: "esm", outfile: output, write: true }); + bundle: true, plugins: [{ name: "resolve-temporary-parser-imports", setup(context) { + context.onResolve({ filter: /^[^./]/ }, args => ({ + path: args.path.startsWith("node:") ? args.path : import.meta.resolve(args.path), external: true, + })); + } }], platform: "node", format: "esm", outfile: output, write: true }); const { program } = await import(pathToFileURL(output)); const commands = [program.name()]; function visit(parent, prefix) { @@ -88,18 +92,25 @@ async function sourceReceipt() { node: process.versions.node }; } +export function runTests(report, json) { + return spawnSync("pnpm", ["test", "--maxWorkers=2", "--reporter=default", "--reporter=json", + "--reporter=./scripts/test-command-reporter.mjs", `--outputFile=${json}`], { + cwd: root, env: { ...process.env, COMMAND_MATRIX_REPORT: report }, stdio: "inherit", + }); +} + +export async function matrixDestination(path) { + return path ? resolve(path) : join(await mkdtemp(join(tmpdir(), "command-matrix-output-")), "matrix.json"); +} + async function main() { - // Inside the repository so external imports resolve the installed dependencies. - const scratch = await mkdtemp(join(root, "scripts/.command-matrix-")); + const scratch = await mkdtemp(join(tmpdir(), "command-matrix-parser-")); const reportRoot = await mkdtemp(join(tmpdir(), "command-matrix-")); try { const commands = await parserCommands(scratch); const report = join(reportRoot,"runner.json"); const json = join(reportRoot,"vitest.json"); - const runner = spawnSync("pnpm", ["test", - "--maxWorkers=2", "--reporter=json", "--reporter=./scripts/test-command-reporter.mjs", `--outputFile=${json}`], { - cwd: root, env: { ...process.env, COMMAND_MATRIX_REPORT: report }, encoding: "utf8", maxBuffer: 40 * 1024 * 1024, - }); + const runner = runTests(report, json); if (runner.error) throw runner.error; const observed = JSON.parse(await readFile(report,"utf8")); const official = JSON.parse(await readFile(json,"utf8")); @@ -109,13 +120,13 @@ async function main() { if (mapping.schemaVersion !== 1) throw new Error("Unsupported mapping schema"); const matrix = reconcile(commands, observed.registered, observed.tests, mapping.links); const output = { schemaVersion: 1, receipt: await sourceReceipt(), - commandCount: commands.length, zeroTestCommands: matrix.rows.filter(row => !row.registered).map(row => row.command), + commandCount: commands.length, unattributedCommands: matrix.rows.filter(row => !row.registered).map(row => row.command), totals: counts(observed.tests), uniqueAttributed: matrix.uniqueAttributed, runnerExitCode: runner.status, moduleErrors: observed.moduleErrors, unhandledErrors: observed.unhandledErrors, commands: matrix.rows, unattributed: matrix.unattributed }; - const destination = resolve(process.argv[2] ?? "docs/test-command-matrix.json"); + const destination = await matrixDestination(process.argv[2]); await writeFile(destination, `${JSON.stringify(output,null,2)}\n`); - console.log(JSON.stringify({ commandCount: output.commandCount, zeroTestCommands: output.zeroTestCommands.length, + console.log(JSON.stringify({ commandCount: output.commandCount, unattributedCommands: output.unattributedCommands.length, ...output.totals, uniqueAttributed: output.uniqueAttributed, output: relative(root,destination) })); if (runner.status !== 0 || observed.moduleErrors.length || observed.unhandledErrors) process.exitCode = 1; } finally { diff --git a/test/command-matrix.test.ts b/test/command-matrix.test.ts index 99b9d9b8..8f96d2aa 100644 --- a/test/command-matrix.test.ts +++ b/test/command-matrix.test.ts @@ -1,6 +1,14 @@ -import { expect, it } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import { spawnSync } from "node:child_process"; +import { expect, it, vi } from "vitest"; // @ts-expect-error The CLI support script is native ESM JavaScript. -import { reconcile, counts } from "../scripts/test-command-matrix.mjs"; +import { reconcile, counts, runTests, matrixDestination, parserCommands } from "../scripts/test-command-matrix.mjs"; + +vi.mock("node:child_process", async importOriginal => ({ + ...await importOriginal(), spawnSync: vi.fn(), +})); it("reconciles shared command attribution without double-counting unique runner IDs", () => { const registered = [{ id: "a", file: "case.ts", name: "parameter 1" }, { id: "b", file: "case.ts", name: "parameter 2" }]; @@ -15,3 +23,34 @@ it("reconciles shared command attribution without double-counting unique runner expect(() => reconcile(["one"], registered, results, [{ file: "case.ts", name: "parameter 1", commands: ["unknown"], evidence: "fixture" }])).toThrow(/Unknown/); expect(counts([{ status: "passed" }, { status: "failed" }, { status: "skipped" }, { status: "pending" }])).toEqual({ registered: 4, executed: 2, pass: 1, fail: 1, skipped: 1, pending: 1 }); }); + +it("rejects mappings without a commands array with an explicit validation error", () => { + const registered = [{ id: "a", file: "case.ts", name: "case" }]; + for (const commands of [undefined, null, "one", []]) { + expect(() => reconcile(["one"], registered, registered, + [{ file: "case.ts", name: "case", evidence: "fixture", commands }])).toThrow(/Invalid mapping/); + } +}); + +it("streams default reporter output and runner errors to CI", () => { + runTests("runner.json", "vitest.json"); + expect(vi.mocked(spawnSync)).toHaveBeenCalledWith("pnpm", expect.arrayContaining(["test", "--reporter=default"]), + expect.objectContaining({ stdio: "inherit" })); +}); + +it("keeps the default report outside the checkout in a temporary directory", async () => { + const output = await matrixDestination(); + try { + expect(relative(tmpdir(), output)).toMatch(/^command-matrix-output-[^/]+\/matrix\.json$/); + expect(await matrixDestination("custom.json")).toBe(join(process.cwd(), "custom.json")); + } finally { await rm(dirname(output), { recursive: true, force: true }); } +}); + +it("loads the Commander tree from a scratch directory outside the checkout", async () => { + const scratch = await mkdtemp(join(tmpdir(), "command-matrix-parser-test-")); + try { + const commands = await parserCommands(scratch); + expect(commands).toEqual(expect.arrayContaining(["agentwheel", "cache prune", "registry publish", "doctor"])); + expect(new Set(commands).size).toBe(commands.length); + } finally { await rm(scratch, { recursive: true, force: true }); } +}); diff --git a/test/invariants/i2-no-loss.test.ts b/test/invariants/i2-no-loss.test.ts index 7f579679..c0c822a9 100644 --- a/test/invariants/i2-no-loss.test.ts +++ b/test/invariants/i2-no-loss.test.ts @@ -1,11 +1,11 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import { expect, it } from "vitest"; import { GitSourceDriver } from "../../src/source/git.js"; import { diffEffects, snapshot } from "../helpers/effects.js"; -it.fails("read_missing_git_preserves_cache", async () => { +async function observeIncompleteCacheRead() { const root = await mkdtemp(join(tmpdir(), "invariant-i2-")); try { const driver = new GitSourceDriver(); @@ -15,8 +15,22 @@ it.fails("read_missing_git_preserves_cache", async () => { await writeFile(join(resolved.resolvedPath, "sentinel"), "preserve me"); const before = await snapshot(root); const failure = await driver.fetch(resolved).then(() => undefined, error => error); - expect(failure).toBeInstanceOf(Error); - // Expected: a read refuses an incomplete checkout without deleting existing bytes. - expect(diffEffects(before, await snapshot(root))).toEqual([]); + const cachePath = relative(root, resolved.resolvedPath); + return { failure, effects: diffEffects(before, await snapshot(root)), + removedPaths: [`tree:${cachePath}`, `tree:${cachePath}/sentinel`] }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +// Expected: a read refuses an incomplete checkout without deleting existing bytes. +it.fails("read_missing_git_preserves_cache", async () => { + const observed = await observeIncompleteCacheRead(); + expect(observed.failure).toBeInstanceOf(Error); + expect(observed.effects).toEqual([]); +}); + +it("read_missing_git_preserves_cache_current_defect_deletes_only_sentinel_tree", async () => { + const observed = await observeIncompleteCacheRead(); + expect(observed.failure).toBeInstanceOf(Error); + expect(observed.effects).toEqual(observed.removedPaths); }); diff --git a/test/invariants/i5-admission-parity.test.ts b/test/invariants/i5-admission-parity.test.ts index b3f446b7..f94853b8 100644 --- a/test/invariants/i5-admission-parity.test.ts +++ b/test/invariants/i5-admission-parity.test.ts @@ -7,7 +7,7 @@ import { targetMappingSchema, type AdapterConfig } from "../../src/model/adapter import { hashPath } from "../../src/utils/fs.js"; import { diffEffects, snapshot } from "../helpers/effects.js"; -it.fails("stale_merge_metadata_rejected_in_all_paths", async () => { +async function observeStaleMergeAdmission() { const root = await mkdtemp(join(tmpdir(), "invariant-i5-")); try { const sourcePath = join(root, "source.json"); @@ -26,12 +26,26 @@ it.fails("stale_merge_metadata_rejected_in_all_paths", async () => { const preview = await createCombinedInstallPlan([artifact], adapter, root, manifest); expect(diffEffects(before, await snapshot(root))).toEqual([]); const failure = await applyCombinedInstallPlan(preview).then(() => undefined, error => error); - expect(failure).toBeInstanceOf(Error); - // Expected: preview blocks the same stale contribution that apply rejects. - expect(preview.hasBlockingChanges).toBe(true); - expect(preview.operations).toEqual(expect.arrayContaining([expect.objectContaining({ - action: "conflict", blockedReason: expect.stringMatching(/contribution|merge/i), - })])); - expect(diffEffects(before, await snapshot(root))).toEqual([]); + return { preview, failure, effects: diffEffects(before, await snapshot(root)) }; } finally { await rm(root, { recursive: true, force: true }); } +} + +// Expected: preview blocks the same stale contribution that apply rejects. +it.fails("stale_merge_metadata_rejected_in_all_paths", async () => { + const { preview, failure, effects } = await observeStaleMergeAdmission(); + expect(failure).toBeInstanceOf(Error); + expect(preview.hasBlockingChanges).toBe(true); + expect(preview.operations).toEqual(expect.arrayContaining([expect.objectContaining({ + action: "conflict", blockedReason: expect.stringMatching(/contribution|merge/i), + })])); + expect(effects).toEqual([]); +}); + +it("stale_merge_metadata_rejected_in_all_paths_current_defect_skips_preview_but_apply_refuses", async () => { + const { preview, failure } = await observeStaleMergeAdmission(); + expect(preview.hasBlockingChanges).toBe(false); + expect(preview.operations).toMatchObject([{ action: "skip" }]); + expect(preview.operations).toHaveLength(1); + expect(failure).toBeInstanceOf(Error); + expect(String(failure)).toMatch(/exact merge contribution/); }); diff --git a/test/invariants/i7-effects-containment.test.ts b/test/invariants/i7-effects-containment.test.ts deleted file mode 100644 index 7a9919fb..00000000 --- a/test/invariants/i7-effects-containment.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { expect, it } from "vitest"; -import { refuseNewPinCollisions } from "../../src/lifecycle/pin-preflight.js"; -import { diffEffects, snapshot } from "../helpers/effects.js"; - -it.fails("pin_provenance_and_feature_floor_enforced", async () => { - const root = await mkdtemp(join(tmpdir(), "invariant-i7-")); - try { - const before = await snapshot(root); - const warnings: string[] = []; - const failure = await refuseNewPinCollisions(root, { - schemaVersion: 1, packages: [{ name: "synthetic-pin", source: join(root, "unresolvable"), mode: "pinned" }], - }, message => warnings.push(message)).then(() => undefined, error => error); - expect(diffEffects(before, await snapshot(root))).toEqual([]); - // Expected: an unresolved approved pin refuses; warning and success are insufficient. - expect({ refused: failure instanceof Error, warnings }).toMatchObject({ refused: true }); - } finally { await rm(root, { recursive: true, force: true }); } -});