From 4d4d7d809477f3505e0f86d95627196cebd8a07c Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:00:54 -0700 Subject: [PATCH 01/10] Optimize compilation, indexing, and memory writes Cache compiled macro grammars and TypeScript library sources, deduplicate term embeddings, and filter vectors before scoring. Avoid redundant hashing and history lookups, and clear script timeout timers after execution. --- .../src/execution/scriptExecutor.ts | 5 +- .../agent-flows/test/scriptExecutor.spec.ts | 39 ++++++++++ .../src/macroAuthoringRuntime.ts | 32 ++++++-- .../copilot-macros/src/recipeCompiler.ts | 52 +++++++++++-- .../test/macroAuthoringRuntime.spec.ts | 6 ++ .../test/macroGrammarCache.spec.ts | 75 +++++++++++++++++++ .../test/recipeCompiler.spec.ts | 22 ++++++ ts/packages/knowPro/src/fuzzyIndex.ts | 5 +- ts/packages/knowPro/src/relatedTermsIndex.ts | 39 ++++++++-- .../knowPro/test/embeddingFilter.spec.ts | 53 +++++++++++++ .../knowPro/test/embeddingRoles.spec.ts | 56 ++++++++++++++ .../memory/service/src/fileMemoryService.ts | 5 +- ts/packages/memory/service/src/viewHistory.ts | 9 ++- .../service/test/viewHistoryEntries.spec.ts | 55 ++++++++++++++ 14 files changed, 429 insertions(+), 24 deletions(-) create mode 100644 ts/packages/agent-flows/test/scriptExecutor.spec.ts create mode 100644 ts/packages/copilot-macros/test/macroGrammarCache.spec.ts create mode 100644 ts/packages/knowPro/test/embeddingFilter.spec.ts create mode 100644 ts/packages/memory/service/test/viewHistoryEntries.spec.ts diff --git a/ts/packages/agent-flows/src/execution/scriptExecutor.ts b/ts/packages/agent-flows/src/execution/scriptExecutor.ts index 667bdf3793..c7775d32b1 100644 --- a/ts/packages/agent-flows/src/execution/scriptExecutor.ts +++ b/ts/packages/agent-flows/src/execution/scriptExecutor.ts @@ -67,6 +67,7 @@ export function createScriptExecutor(config: ScriptExecutorConfig): { ...blockedGlobalsOverride, }; + let timeoutId: ReturnType | undefined; try { const fn = new Function( ...Object.keys(sandbox), @@ -77,7 +78,7 @@ export function createScriptExecutor(config: ScriptExecutorConfig): { const timeout = options?.timeout ?? defaultTimeout; const timeoutPromise = new Promise((_, reject) => { - setTimeout( + timeoutId = setTimeout( () => reject(new Error("Script execution timeout")), timeout, ); @@ -112,6 +113,8 @@ export function createScriptExecutor(config: ScriptExecutorConfig): { error: errorMessage, message: `Script execution failed: ${errorMessage}`, }; + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); } }, }; diff --git a/ts/packages/agent-flows/test/scriptExecutor.spec.ts b/ts/packages/agent-flows/test/scriptExecutor.spec.ts new file mode 100644 index 0000000000..ec6c165c41 --- /dev/null +++ b/ts/packages/agent-flows/test/scriptExecutor.spec.ts @@ -0,0 +1,39 @@ +import { jest } from "@jest/globals"; +import { createScriptExecutor } from "../src/execution/scriptExecutor.js"; + +describe("script execution timers", () => { + const executor = createScriptExecutor({ + apiParamName: "api", + defaultTimeout: 100, + }); + + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + test.each([ + ["async () => 42", true], + ['async () => { throw new Error("failed"); }', false], + ['() => { throw new Error("failed"); }', false], + ["not valid script", false], + ])("clears timers after %s", async (source, success) => { + const result = await executor.execute(source, {}, {}); + expect(result.success).toBe(success); + expect(jest.getTimerCount()).toBe(0); + }); + + test("still times out a pending script and clears the timer", async () => { + const result = executor.execute( + "async (api) => api.pending()", + { pending: () => new Promise(() => {}) }, + {}, + { timeout: 20 }, + ); + expect(jest.getTimerCount()).toBe(1); + await jest.advanceTimersByTimeAsync(20); + expect(await result).toMatchObject({ + success: false, + error: "Script execution timeout", + }); + expect(jest.getTimerCount()).toBe(0); + }); +}); diff --git a/ts/packages/copilot-macros/src/macroAuthoringRuntime.ts b/ts/packages/copilot-macros/src/macroAuthoringRuntime.ts index 9eb593978c..210ccbd48c 100644 --- a/ts/packages/copilot-macros/src/macroAuthoringRuntime.ts +++ b/ts/packages/copilot-macros/src/macroAuthoringRuntime.ts @@ -916,13 +916,35 @@ function validMacroParameters( }); } +const compiledMacroGrammars = new Map< + string, + { + grammar: ReturnType; + nfa: ReturnType; + } +>(); +const maxCompiledMacroGrammars = 128; + function compileMacroGrammar(rules: string[]) { + const source = rules.join("\n"); + const cached = compiledMacroGrammars.get(source); + if (cached) { + compiledMacroGrammars.delete(source); + compiledMacroGrammars.set(source, cached); + return cached; + } try { - const grammar = loadGrammarRules( - "learned-macros.agr", - rules.join("\n"), - ); - return { grammar, nfa: compileGrammarToNFA(grammar, "macros") }; + const grammar = loadGrammarRules("learned-macros.agr", source); + const compiled = { + grammar, + nfa: compileGrammarToNFA(grammar, "macros"), + }; + compiledMacroGrammars.set(source, compiled); + if (compiledMacroGrammars.size > maxCompiledMacroGrammars) { + const oldest = compiledMacroGrammars.keys().next().value; + if (oldest !== undefined) compiledMacroGrammars.delete(oldest); + } + return compiled; } catch { throw new Error( "Macro grammar rejected (stage=compile; matchCount=unknown).", diff --git a/ts/packages/copilot-macros/src/recipeCompiler.ts b/ts/packages/copilot-macros/src/recipeCompiler.ts index 72255208b7..b47f1f008f 100644 --- a/ts/packages/copilot-macros/src/recipeCompiler.ts +++ b/ts/packages/copilot-macros/src/recipeCompiler.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import ts from "typescript"; -import { join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import type { RecipeArtifact } from "./recipeContracts.js"; import { recipeRuntimeAbi } from "./recipeContracts.js"; import { recipeDigest } from "./recipeValidation.js"; @@ -37,6 +37,10 @@ export const recipeCompilerIdentity = Object.freeze({ }); const checkedExecutables = new Map(); +const librarySources = new Map< + string, + { text: string; parserOptions: string; source: ts.SourceFile } +>(); let libraryDirectory: string | undefined; @@ -46,6 +50,7 @@ export function configureRecipeCompilerAssets(assets: { }): void { libraryDirectory = assets.libraryDirectory; checkedExecutables.clear(); + librarySources.clear(); } function remember(sourceDigest: string, executableDigest: string): void { @@ -169,10 +174,47 @@ export function compileRecipeProgram( join(directory, ts.getDefaultLibFileName(options)); } const originalGetSource = host.getSourceFile.bind(host); - host.getSourceFile = (name, languageVersion, onError, shouldCreate) => - name === "recipe.ts" - ? ts.createSourceFile(name, text, languageVersion, true) - : originalGetSource(name, languageVersion, onError, shouldCreate); + host.getSourceFile = (name, languageVersion, onError, shouldCreate) => { + if (name === "recipe.ts") + return ts.createSourceFile(name, text, languageVersion, true); + if ( + shouldCreate || + dirname(resolve(name)) !== resolve(host.getDefaultLibLocation!()) || + !/^lib\..*\.d\.ts$/.test(basename(name)) + ) + return originalGetSource( + name, + languageVersion, + onError, + shouldCreate, + ); + const libraryText = host.readFile(name); + const parserOptions = JSON.stringify(languageVersion); + const key = resolve(name); + const cached = librarySources.get(key); + if ( + libraryText !== undefined && + cached?.text === libraryText && + cached.parserOptions === parserOptions + ) + return cached.source; + const source = originalGetSource( + name, + languageVersion, + onError, + shouldCreate, + ); + if (source && libraryText !== undefined) { + if (librarySources.size >= 128) + librarySources.delete(librarySources.keys().next().value!); + librarySources.set(key, { + text: libraryText, + parserOptions, + source, + }); + } + return source; + }; let javascript: string | undefined; host.writeFile = (name, content) => { if (name.endsWith(".js")) javascript = content; diff --git a/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts b/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts index 3966b0ed6a..d995ffaaa6 100644 --- a/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts +++ b/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts @@ -288,6 +288,12 @@ describe("portable authoring boundary", () => { expect(matchMacroGrammarInputs(approved, "read beta now")).toEqual([ { path: "beta" }, ]); + expect(matchMacroGrammarInputs(approved, "read gamma now")).toEqual( + [{ path: "gamma" }], + ); + expect(matchMacroGrammarInputs(approved, "read beta now")).toEqual([ + { path: "beta" }, + ]); expect(matchMacroGrammarInputs(macro, "read beta now")).toEqual([]); expect( matchMacroGrammarInputs(approved, "delete beta now"), diff --git a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts new file mode 100644 index 0000000000..8d918eee56 --- /dev/null +++ b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts @@ -0,0 +1,75 @@ +import type { CopilotToolMacro } from "../src/contracts.js"; + +const jest = ( + import.meta as ImportMeta & { + jest: typeof globalThis.jest & { + unstable_mockModule(name: string, factory: () => unknown): void; + }; + } +).jest; +const runtime = await import("@typeagent/action-grammar/runtime"); +const compile = jest.fn(runtime.compileGrammarToNFA); +jest.unstable_mockModule("@typeagent/action-grammar/runtime", () => ({ + ...runtime, + compileGrammarToNFA: compile, +})); +const { matchMacroGrammarInputs } = await import( + "../src/macroAuthoringRuntime.js" +); + +function macro(phrase: string): CopilotToolMacro { + return { + schemaVersion: 1, + macroId: "cached", + version: 1, + name: "Cached", + description: "Read fixture", + state: "approved", + executionClass: "replayable", + inputs: [], + steps: [], + sourceTraceId: "fixture", + createdAt: "2026-10-18T00:00:00Z", + warnings: [], + learning: { + jobId: "fixture", + cwd: ".", + mode: "read-only", + requiresLivePermissions: true, + grammarRules: [ + ` = ${phrase} -> { actionName: "run_cached_v1" };`, + ], + exampleInputs: {}, + requests: [phrase], + }, + }; +} + +test("reuses exact grammar source, detects edits and evicts old compilations", () => { + const original = macro("read fixture"); + expect(matchMacroGrammarInputs(original, "read fixture")).toEqual([{}]); + expect( + matchMacroGrammarInputs(structuredClone(original), "read fixture"), + ).toEqual([{}]); + expect(compile).toHaveBeenCalledTimes(1); + const changed = macro("read updated"); + expect(matchMacroGrammarInputs(changed, "read fixture")).toEqual([]); + expect(matchMacroGrammarInputs(changed, "read updated")).toEqual([{}]); + expect(compile).toHaveBeenCalledTimes(2); + for (let ordinal = 0; ordinal < 128; ordinal++) { + const phrase = `read fixture${ordinal}`; + expect(matchMacroGrammarInputs(macro(phrase), phrase)).toEqual([{}]); + } + const beforeReload = compile.mock.calls.length; + expect(matchMacroGrammarInputs(original, "read fixture")).toEqual([{}]); + expect(compile).toHaveBeenCalledTimes(beforeReload + 1); +}); + +test("failed compilations do not populate the cache", () => { + const valid = macro("read repaired"); + compile.mockImplementationOnce(() => { + throw new Error("Injected compiler failure"); + }); + expect(matchMacroGrammarInputs(valid, "read repaired")).toEqual([]); + expect(matchMacroGrammarInputs(valid, "read repaired")).toEqual([{}]); +}); diff --git a/ts/packages/copilot-macros/test/recipeCompiler.spec.ts b/ts/packages/copilot-macros/test/recipeCompiler.spec.ts index 801765ede3..2365f0c585 100644 --- a/ts/packages/copilot-macros/test/recipeCompiler.spec.ts +++ b/ts/packages/copilot-macros/test/recipeCompiler.spec.ts @@ -18,6 +18,28 @@ describe("recipe program compiler", () => { expect(first.compiler.name).toBe("typescript"); }); + it("keeps recipe diagnostics and declarations isolated across warm compilations", () => { + const valid = `async function execute(api, params) { + const value: Map = new Map(); + value.set("entry", 1); + return value.get("entry") ?? 0; + }`; + const cold = compileRecipeProgram(valid); + expect(() => + compileRecipeProgram(`async function execute(api, params) { + return value; + }`), + ).toThrow("Cannot find name 'value'"); + expect(() => + compileRecipeProgram(`async function execute(api, params) { + const value: Map = new Map(); + value.set("entry", "wrong"); + return null; + }`), + ).toThrow(); + expect(compileRecipeProgram(valid)).toEqual(cold); + }); + it.each([ `async function execute(api, params) { const value: number = "wrong"; return value; }`, `async function execute(api, params) { return api.tools.missing("read", null); }`, diff --git a/ts/packages/knowPro/src/fuzzyIndex.ts b/ts/packages/knowPro/src/fuzzyIndex.ts index d7f05d4e4c..38edc18fa6 100644 --- a/ts/packages/knowPro/src/fuzzyIndex.ts +++ b/ts/packages/knowPro/src/fuzzyIndex.ts @@ -168,8 +168,11 @@ export class EmbeddingIndex { messageOrdinal < list.length; ++messageOrdinal ) { + if (!predicate(messageOrdinal)) { + continue; + } const score: number = dotProduct(list[messageOrdinal], other); - if (score >= minScore && predicate(messageOrdinal)) { + if (score >= minScore) { matches.push(messageOrdinal, score); } } diff --git a/ts/packages/knowPro/src/relatedTermsIndex.ts b/ts/packages/knowPro/src/relatedTermsIndex.ts index bc71f275aa..f4ba981aba 100644 --- a/ts/packages/knowPro/src/relatedTermsIndex.ts +++ b/ts/packages/knowPro/src/relatedTermsIndex.ts @@ -330,6 +330,7 @@ export class TermEmbeddingIndex implements ITermEmbeddingIndex, TextEmbeddingCache { private textArray: string[]; + private textPositions = new Map(); private embeddingIndex: TextEmbeddingIndex; constructor( @@ -348,14 +349,27 @@ export class TermEmbeddingIndex terms: string[], eventHandler?: IndexingEventHandlers, ): Promise { + const newTerms = [...new Set(terms)].filter( + (term) => !this.textPositions.has(term), + ); + if (newTerms.length === 0) return { numberCompleted: terms.length }; const result = await this.embeddingIndex.addTextBatch( - terms, + newTerms, eventHandler, ); if (result.numberCompleted > 0) { - this.textArray.push(...terms.slice(0, result.numberCompleted)); + for (const term of newTerms.slice(0, result.numberCompleted)) { + this.textPositions.set(term, this.textArray.length); + this.textArray.push(term); + } } - return result; + const incomplete = terms.findIndex( + (term) => !this.textPositions.has(term), + ); + return { + ...result, + numberCompleted: incomplete < 0 ? terms.length : incomplete, + }; } public async lookupTerm( @@ -390,15 +404,17 @@ export class TermEmbeddingIndex } public removeTerm(term: string): void { - const indexOf = this.textArray.indexOf(term); - if (indexOf >= 0) { + const indexOf = this.textPositions.get(term); + if (indexOf !== undefined) { this.textArray.splice(indexOf, 1); this.embeddingIndex.removeAt(indexOf); + this.rebuildTextPositions(); } } public clear(): void { this.textArray = []; + this.textPositions.clear(); this.embeddingIndex.clear(); } @@ -416,6 +432,7 @@ export class TermEmbeddingIndex ); } this.textArray = data.textItems; + this.rebuildTextPositions(); this.embeddingIndex.deserialize(data.embeddings); } @@ -437,13 +454,21 @@ export class TermEmbeddingIndex return undefined; } } - const pos = this.textArray.indexOf(text); - if (pos >= 0) { + const pos = this.textPositions.get(text); + if (pos !== undefined) { return serializeEmbedding(this.embeddingIndex.get(pos)); } return undefined; } + private rebuildTextPositions(): void { + this.textPositions.clear(); + this.textArray.forEach((text, ordinal) => { + if (!this.textPositions.has(text)) + this.textPositions.set(text, ordinal); + }); + } + private matchesToTerms(matches: Scored[]): Term[] { return matches.map((m) => { return { text: this.textArray[m.item], weight: m.score }; diff --git a/ts/packages/knowPro/test/embeddingFilter.spec.ts b/ts/packages/knowPro/test/embeddingFilter.spec.ts new file mode 100644 index 0000000000..f893177d2a --- /dev/null +++ b/ts/packages/knowPro/test/embeddingFilter.spec.ts @@ -0,0 +1,53 @@ +import { EmbeddingIndex } from "../src/fuzzyIndex.js"; + +describe("filtered embedding search", () => { + const query = new Float32Array([1, 0]); + const vectors = [ + new Float32Array([1, 0]), + new Float32Array([0.8, 0.6]), + new Float32Array([0.6, 0.8]), + new Float32Array([0, 1]), + ]; + + test.each([undefined, 0, 1, 2, 10])( + "preserves eligible scores and ordering with maxMatches=%s", + (maxMatches) => { + const index = new EmbeddingIndex(vectors); + const eligible = (ordinal: number) => ordinal % 2 === 1; + const expected = index + .getIndexesOfNearest(query, undefined, 0.1) + .filter((match) => eligible(match.item)); + expect( + index.getIndexesOfNearest(query, maxMatches, 0.1, eligible), + ).toEqual( + maxMatches && maxMatches > 0 + ? expected.slice(0, maxMatches) + : expected, + ); + }, + ); + + test("does not access vectors rejected by the predicate", () => { + const guardedVectors = [...vectors]; + Object.defineProperty(guardedVectors, 0, { + get() { + throw new Error("Rejected vector was scored"); + }, + }); + const index = new EmbeddingIndex(guardedVectors); + expect( + index.getIndexesOfNearest(query, 1, 0, (ordinal) => ordinal !== 0), + ).toEqual([{ item: 1, score: vectors[1][0] }]); + }); + + test("an empty filter returns no matches", () => { + expect( + new EmbeddingIndex(vectors).getIndexesOfNearest( + query, + undefined, + undefined, + () => false, + ), + ).toEqual([]); + }); +}); diff --git a/ts/packages/knowPro/test/embeddingRoles.spec.ts b/ts/packages/knowPro/test/embeddingRoles.spec.ts index 49a1217a54..38ba97b778 100644 --- a/ts/packages/knowPro/test/embeddingRoles.spec.ts +++ b/ts/packages/knowPro/test/embeddingRoles.spec.ts @@ -154,4 +154,60 @@ describe("KnowPro embedding roles (synthetic/offline)", () => { await index.addTerms(["term"]); expect(index.getEmbedding("term")).toEqual([1, 0]); }); + + test("term additions deduplicate exact text across batches and reloads", async () => { + const settings = createTextEmbeddingIndexSettings(model(), 2); + const index = new TermEmbeddingIndex(settings); + expect( + (await index.addTerms(["one", "one", "two"])).numberCompleted, + ).toBe(3); + expect((await index.addTerms(["two", "three"])).numberCompleted).toBe( + 2, + ); + expect(calls).toEqual(["document:one,two", "document:three"]); + const restored = new TermEmbeddingIndex(settings, index.serialize()); + await restored.addTerms(["one", "three"]); + expect(calls).toHaveLength(2); + restored.removeTerm("two"); + expect( + restored.getEmbedding( + embeddingCacheKey("three", identity("document")), + ), + ).toEqual([1, 0]); + expect( + restored.getEmbedding( + embeddingCacheKey("two", identity("document")), + ), + ).toBeUndefined(); + restored.clear(); + expect( + restored.getEmbedding( + embeddingCacheKey("one", identity("document")), + ), + ).toBeUndefined(); + await restored.addTerms(["one"]); + expect(restored.serialize().textItems).toEqual(["one"]); + }); + + test("partial vocabulary additions report the original completed prefix and retry missing terms", async () => { + const settings = createTextEmbeddingIndexSettings(model(), 2); + settings.batchSize = 1; + const index = new TermEmbeddingIndex(settings); + await index.addTerms(["existing"]); + const result = await index.addTerms( + ["existing", "one", "one", "two", "existing"], + { onEmbeddingsCreated: (_all, _batch, startAt) => startAt === 0 }, + ); + expect(result.numberCompleted).toBe(3); + expect(index.serialize().textItems).toEqual(["existing", "one"]); + expect( + (await index.addTerms(["one", "two", "existing"])).numberCompleted, + ).toBe(3); + expect(index.serialize().textItems).toEqual(["existing", "one", "two"]); + expect(calls).toEqual([ + "document:existing", + "document:one", + "document:two", + ]); + }); }); diff --git a/ts/packages/memory/service/src/fileMemoryService.ts b/ts/packages/memory/service/src/fileMemoryService.ts index 1732c80f1b..fdf0dc9913 100644 --- a/ts/packages/memory/service/src/fileMemoryService.ts +++ b/ts/packages/memory/service/src/fileMemoryService.ts @@ -1910,8 +1910,9 @@ export class FileMemoryService ); } const content = contentFor(request); - const contentHash = request.source.contentHash ?? hashContent(content); - if (contentHash !== hashContent(content)) { + const computedContentHash = hashContent(content); + const contentHash = request.source.contentHash ?? computedContentHash; + if (contentHash !== computedContentHash) { throw new Error("The supplied content hash does not match content"); } const sourceId = request.source.sourceId ?? randomUUID(); diff --git a/ts/packages/memory/service/src/viewHistory.ts b/ts/packages/memory/service/src/viewHistory.ts index 07afee1898..b08115b0d7 100644 --- a/ts/packages/memory/service/src/viewHistory.ts +++ b/ts/packages/memory/service/src/viewHistory.ts @@ -163,6 +163,9 @@ export class ViewHistory { const previous = head ? (await git.readTree({ ...this.options(), oid: head })).tree : []; + const previousByPath = new Map( + previous.map((entry) => [entry.path, entry]), + ); const entries = []; for (const [name, text] of Object.entries({ "state.json": JSON.stringify(state), @@ -172,9 +175,9 @@ export class ViewHistory { ...this.options(), blob: Buffer.from(text, "utf8"), }); - const unchanged = previous.find( - (entry) => entry.path === name && entry.oid === oid, - ); + const previousEntry = previousByPath.get(name); + const unchanged = + previousEntry?.oid === oid ? previousEntry : undefined; entries.push( unchanged ?? { mode: "100644", diff --git a/ts/packages/memory/service/test/viewHistoryEntries.spec.ts b/ts/packages/memory/service/test/viewHistoryEntries.spec.ts new file mode 100644 index 0000000000..5625e3f902 --- /dev/null +++ b/ts/packages/memory/service/test/viewHistoryEntries.spec.ts @@ -0,0 +1,55 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import fs from "node:fs"; +import git from "isomorphic-git"; +import { ViewHistory } from "../src/viewHistory.js"; + +test("history commits preserve unchanged blobs, updates, removals and old snapshots", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "view-entries-")); + try { + const history = new ViewHistory(directory, () => ({ version: 0 })); + const options = { + fs, + gitdir: path.join(directory, "view-history.git"), + }; + const first = await history.commit( + null, + { version: 1 }, + { "keep.md": "Keep", "change.md": "Old", "remove.md": "Remove" }, + "test", + "first", + ); + const second = await history.commit( + first, + { version: 2 }, + { "keep.md": "Keep", "change.md": "New", "add.md": "Added" }, + "test", + "second", + ); + const previous = (await git.readTree({ ...options, oid: first })).tree; + const current = (await git.readTree({ ...options, oid: second })).tree; + expect(current.find((entry) => entry.path === "keep.md")).toEqual( + previous.find((entry) => entry.path === "keep.md"), + ); + expect(current.map((entry) => entry.path).sort()).toEqual([ + "add.md", + "change.md", + "keep.md", + "state.json", + ]); + const changed = await git.readBlob({ + ...options, + oid: second, + filepath: "change.md", + }); + expect(Buffer.from(changed.blob).toString("utf8")).toBe("New"); + expect((await history.read(first)).state).toEqual({ version: 1 }); + expect((await history.read()).state).toEqual({ version: 2 }); + await expect( + history.commit(first, { version: 3 }, {}, "test", "stale"), + ).rejects.toThrow("head conflict"); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); From 96738560ee3b3dfad028c6a03fbb52c98f008f2d Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:43:44 -0700 Subject: [PATCH 02/10] Improve memory and macro performance with incremental storage - Coalesce batch ingestion and index only changed source revisions - Store revision text, indexes, and events in immutable segments - Read View history without decoding unrelated corpus snapshots - Reduce macro record rewrites and replace authoring polling with IPC - Preserve citation eligibility, durable checkpoints, and privacy cleanup - Add benchmarks, recovery tests, and implementation documentation --- ts/docs/architecture/memory/current-system.md | 78 ++ ts/docs/architecture/workflows/macros.md | 68 + ts/packages/copilot-macros/src/contracts.ts | 3 + .../copilot-macros/src/macroChildAuthoring.ts | 111 +- .../copilot-macros/src/macroLearning.ts | 257 ++-- .../copilot-macros/src/macroManager.ts | 155 ++- .../src/macroProgramAuthoring.ts | 49 +- .../copilot-macros/src/macroProgramRun.ts | 115 +- .../copilot-macros/src/recipeContracts.ts | 4 + .../copilot-macros/src/recipeExecutor.ts | 298 +++- .../src/recipeExecutorWorker.ts | 254 ++-- .../copilot-macros/src/recipeWorkerPool.ts | 201 +++ .../copilot-macros/src/resultEvidence.ts | 22 +- .../copilot-macros/src/revisionRecords.ts | 242 ++++ .../test/learningEvidence.spec.ts | 21 + .../test/macroAuthoringRuntime.spec.ts | 2 +- .../test/macroChildAuthoring.spec.ts | 140 +- .../test/macroChildLearning.spec.ts | 38 +- .../copilot-macros/test/macroLearning.spec.ts | 74 +- .../copilot-macros/test/macroPrograms.spec.ts | 305 +++- .../copilot-macros/test/macroStorage.spec.ts | 513 +++++++ .../test/recipeExecutor.spec.ts | 223 +++ .../test/recipeWorkerPool.spec.ts | 404 ++++++ .../src/extension/macro-authoring.ts | 43 +- .../src/extension/native-recipe-service.ts | 9 +- .../src/extension/recipe-native-host.ts | 95 +- .../test/macroAuthoringWorker.spec.ts | 71 +- .../test/recipeNativeHost.spec.ts | 110 +- ts/packages/knowPro/src/index.ts | 1 + ts/packages/knowPro/src/search.ts | 29 +- ts/packages/knowPro/src/segmentedJson.ts | 330 +++++ ts/packages/knowPro/src/serialization.ts | 99 +- .../knowPro/test/embeddingRoles.spec.ts | 84 ++ .../knowPro/test/segmentedJson.spec.ts | 181 +++ .../test/serializationIdentity.spec.ts | 37 + ts/packages/memory/conversation/src/common.ts | 2 + .../memory/conversation/src/docMemory.ts | 2 + ts/packages/memory/service/README.md | 117 +- ts/packages/memory/service/src/batchImport.ts | 76 +- .../service/src/boundedIngestionGroup.ts | 57 + .../memory/service/src/eventSegmentStore.ts | 418 ++++++ .../memory/service/src/fileMemoryService.ts | 1236 +++++++++++++---- ts/packages/memory/service/src/indexSchema.ts | 10 + .../memory/service/src/knowProCorpusIndex.ts | 175 ++- .../memory/service/src/revisionTextStore.ts | 200 +++ ts/packages/memory/service/src/types.ts | 5 + ts/packages/memory/service/src/viewHistory.ts | 130 +- .../memory/service/src/viewHistoryRecords.ts | 58 + ts/packages/memory/service/src/viewStore.ts | 46 +- .../service/test/batchIngestion.spec.ts | 663 +++++++++ .../memory/service/test/eventKnowPro.spec.ts | 25 +- .../service/test/eventSegmentStore.spec.ts | 189 +++ .../service/test/fileMemoryService.spec.ts | 25 +- .../service/test/markdownPersistence.spec.ts | 151 ++ .../service/test/readConcurrency.spec.ts | 225 ++- .../service/test/revisionTextStore.spec.ts | 423 ++++++ .../memory/service/test/topicSnapshot.spec.ts | 78 ++ .../memory/service/test/viewDrafts.spec.ts | 15 + .../service/test/viewHistoryRecords.spec.ts | 550 ++++++++ .../service/test/viewStoreSegments.spec.ts | 106 ++ .../hot-path-benchmark.mjs | 304 ++++ .../macro-runtime-benchmark.mjs | 241 ++++ .../storage-write-benchmark.mjs | 547 ++++++++ .../test/macroRuntimeBenchmark.spec.mjs | 58 + .../test/memoryHotPathBenchmark.spec.mjs | 60 + .../test/storageWriteBenchmark.spec.mjs | 97 ++ 66 files changed, 10083 insertions(+), 872 deletions(-) create mode 100644 ts/packages/copilot-macros/src/recipeWorkerPool.ts create mode 100644 ts/packages/copilot-macros/src/revisionRecords.ts create mode 100644 ts/packages/copilot-macros/test/macroStorage.spec.ts create mode 100644 ts/packages/copilot-macros/test/recipeWorkerPool.spec.ts create mode 100644 ts/packages/knowPro/src/segmentedJson.ts create mode 100644 ts/packages/knowPro/test/segmentedJson.spec.ts create mode 100644 ts/packages/memory/service/src/boundedIngestionGroup.ts create mode 100644 ts/packages/memory/service/src/eventSegmentStore.ts create mode 100644 ts/packages/memory/service/src/revisionTextStore.ts create mode 100644 ts/packages/memory/service/src/viewHistoryRecords.ts create mode 100644 ts/packages/memory/service/test/batchIngestion.spec.ts create mode 100644 ts/packages/memory/service/test/eventSegmentStore.spec.ts create mode 100644 ts/packages/memory/service/test/revisionTextStore.spec.ts create mode 100644 ts/packages/memory/service/test/viewHistoryRecords.spec.ts create mode 100644 ts/packages/memory/service/test/viewStoreSegments.spec.ts create mode 100644 ts/tools/memory-maintenance-qualification/hot-path-benchmark.mjs create mode 100644 ts/tools/memory-maintenance-qualification/macro-runtime-benchmark.mjs create mode 100644 ts/tools/memory-maintenance-qualification/storage-write-benchmark.mjs create mode 100644 ts/tools/scripts/test/macroRuntimeBenchmark.spec.mjs create mode 100644 ts/tools/scripts/test/memoryHotPathBenchmark.spec.mjs create mode 100644 ts/tools/scripts/test/storageWriteBenchmark.spec.mjs diff --git a/ts/docs/architecture/memory/current-system.md b/ts/docs/architecture/memory/current-system.md index a3ed76f569..4b05e77374 100644 --- a/ts/docs/architecture/memory/current-system.md +++ b/ts/docs/architecture/memory/current-system.md @@ -149,6 +149,19 @@ See [History and forgetting](derived-views.md#history-and-forgetting). Coverage: [source acquisition](https://github.com/microsoft/TypeAgent/blob/main/ts/packages/memory/service/test/sourceAcquisition.spec.ts), [maintenance foundation](https://github.com/microsoft/TypeAgent/blob/main/ts/packages/memory/service/test/maintenanceFoundation.spec.ts). +Durable acquired batches coalesce eligible distinct new sources into detached +two-member generations, waiting at most 25 ms for the second durable admission. +One compatible index copy/append and manifest publication serve both jobs; +standalone ingestion and unsupported/replacement/asset/guarded cases retain +singleton behavior. Each member keeps its own job, progress and retry identity. +Cancellation or a stale generation discards the uncommitted candidate and retries +unaffected members; clear cancels captured admissions. Forget/clear and close +drain actual provider writes, even when a provider ignores abort. Physical batch +reads and atomic publications are serialized per batch to avoid Windows +read/rename contention without weakening durable reads. +Coverage: [batch ingestion](https://github.com/microsoft/TypeAgent/blob/main/ts/packages/memory/service/test/batchIngestion.spec.ts), +[KnowPro citation persistence](https://github.com/microsoft/TypeAgent/blob/main/ts/packages/memory/service/test/markdownPersistence.spec.ts). + ## Search and answer semantics | Surface | Result | @@ -206,6 +219,59 @@ The service-owned store holds source manifests and revisions, event ledgers, ingestion jobs and derived index generations. Revision assets and acquired-batch state are stored separately from Views. +Canonical document revision text is stored in corpus-local content-addressed +JSON-string blobs under `revision-text`. Version-1 corpus manifests hold digest +and byte-length references instead of inline content. New blobs are verified +before atomic manifest replacement; existing immutable revision references do +not rewrite retained text. Startup verifies referenced blobs and collects +unreferenced blobs/staging files. Missing/corrupt canonical text is an error, not +a reason to rebuild it from a derived index. + +Manifest references are the only retention authority. Shared text survives while +another retained source/revision references it; forgetting the final reference +physically removes the blob before reopening reads. Corpus clear now has a +durable purge intent, like source forget, and restart finishes interrupted +cleanup. Incompatible canonical formats require explicit reset/reimport; no +canonical data is silently migrated or deleted. This removes text rewrite +amplification, not metadata scaling or in-memory text retention. Atomic +replacement does not establish power-loss durability. + +KnowPro derived generations publish small descriptors referencing shared +immutable JSON nodes under their root's `.segments`. Copying a generation no +longer copies all retained node payloads. Compatible retain-history replacements +without acquired assets append only changed content and persist active revision +eligibility; inactive knowledge/vector candidates are excluded before TopN. +Historical reactivation retains message ordinals without re-extraction. +Replace-without-history and unsupported custom indexes still rebuild. Explicit +reindexing compacts inactive indexed revisions. Full hydration, JSON traversal, +metadata work and linear vector search remain. + +Canonical events publish a versioned `events.jsonl` descriptor over immutable +`event-segments` payloads, a monotonic sequence and independent eligibility +revision. Cold single-event lookup reads its body plus suppression metadata, +not all event bodies, but still scans filenames. Listing/search load the +required event set. Appending an event incrementally extends a compatible +derived index; deletion invalidates the old index even if a later append +restores the previous count. Interrupted physical event/index purge withholds +reads until recovery finishes. Older canonical event formats require explicit +reset/reimport; derived indexes can be rebuilt. + +Explicit document reindexing captures the source revisions under a short corpus +reservation, prepares a new generation outside the reservation, then publishes +under a second reservation. Publication rejects a changed source or index +generation instead of overwriting newer work. Readers keep using the committed +generation while preparation runs; publication waits for existing readers. +Document search repair follows the same detached path and coalesces concurrent +readers' repair requests. Repair failures are returned to callers and can be +retried. Other lock-held maintenance and mutation paths retain their existing +index preparation behavior. + +Forgetting and clearing drain detached provider writes before purging their +candidate directories. Close cancels publication and waits for those writes too. +A provider that ignores cancellation can therefore delay cleanup; the service +does not report successful cleanup while that provider could still restore +removed files. + Each corpus's `views` directory contains a private bare repository, `view-history.git`. Its committed snapshots hold typed Views, generated bases, edits, conflicts, publication and maintenance state, and Topics. The ref @@ -213,6 +279,18 @@ edits, conflicts, publication and maintenance state, and Topics. The ref search indexes are kept apart from it, and the conversation registry and replay metadata are not a second content store. +Full-fidelity per-View records in Git let history queries decode only the +selected View's distinct version arrays, rather than every corpus snapshot. +Ordering, revision filtering, editorial metadata and head guards are unchanged. +Legacy derived records rebuild under lock once; Git metadata traversal still +scales with commits. History privacy replacement removes the records and old +object database together. + +Topic read I/O receipts use the actual committed state blob byte length. The +Topic-only byte length is cached by corpus and commit, rather than serializing +the full state again for every query. This changes neither the committed format +nor Topic eligibility or continuation checks. + See [View history](derived-views.md#history-and-forgetting) and the [service operational guide](https://github.com/microsoft/TypeAgent/blob/main/ts/packages/memory/service/README.md). diff --git a/ts/docs/architecture/workflows/macros.md b/ts/docs/architecture/workflows/macros.md index 52bd6878ce..ff39711384 100644 --- a/ts/docs/architecture/workflows/macros.md +++ b/ts/docs/architecture/workflows/macros.md @@ -168,6 +168,24 @@ provider flows use. CPU and time, guest memory, stack, JSON size and broker call all have limits. The guest reaches the outside world only through declared capability and semantic adapters, never through ambient process APIs. +An owned `RecipeExecutor` keeps at most eight workers warm, with at most 64 +queued admissions. This preserves the eight-way concurrency measured by the +offline runner; higher concurrency waits rather than creating unbounded workers. +Admission time is included in the execution deadline, and excess queued work +fails explicitly. `performance.workerQueueMs` and `performance.workerReused` in +the run result report admitted worker waits and reuse. Idle workers are unrefed +and retire after 30 seconds. + +Workers retain the initialized QuickJS WASM module, not a guest runtime or +context. Every invocation allocates and disposes fresh guest state before +reporting success. Failure, timeout, crash, invalid protocol or failed cleanup +retires the worker. Asset reconfiguration retires idle workers immediately and +active workers after their captured invocation. Closing an executor cancels +active and queued work, waits for termination, and rejects new work. The native +plugin owns an executor for its local service; fixture validation owns a +single-worker executor for one private fixture batch and closes it even on +failure. The standalone execution helper uses a shared, idle-expiring executor. + - **Symbolic** programs make no model calls when they run again. - **Hybrid** programs make only the semantic calls they declare, with bounds, through the configured provider. Results are schema-validated, calls can be @@ -185,6 +203,56 @@ writes, unsupported schemas and changed contracts all fail with an error. Classifying a tool's effect **doesn't grant permission**. Native hooks, content exclusions and the usual host and backend authorization still apply. +Capability preflight uses batches of at most four concurrent inspections and +reports failures in declaration order. Each native execution host captures one +immutable tool inventory and reuses its approved descriptors only within that +binding. Live session/account/workspace/generation checks still run before +inspection and dispatch; permission or tool-change events invalidate the binding. +Failed inventory discovery is not cached. The inventory is not a permission +cache, and each dispatch still goes through native authorization. + +Guest requests and replies are correlated by request ID, so independent +explicitly read-only calls can complete out of order. At most four reads are +active per invocation. Semantic and effectful calls are serialized barriers +between read groups; this does not enable provider-flow concurrency. Denial or +cancellation stops unsent work, aborts supported in-flight waits, and suppresses +late replies. Already dispatched calls may still be running when a native tool +has no exact cancellation API. + +Durable journal mutations remain serialized, including each pre-dispatch +authorization/revision check and flush. Journals now append flushed checkpoints +instead of renaming the full accumulated run on every phase. An exclusive run +claim preserves identity; first-call order and latest-phase maps rebuild from +the journal. A partial final append is ignored conservatively, malformed +complete records fail, and uncertain dispatched work is never automatically +retried. Failed append prevents later dispatch; pending mutations drain before +a run returns. + +The active file-backed manager uses small per-macro catalog pointers and +immutable version/run payloads. Learning and child-authoring jobs use immutable +content-addressed records behind versioned pointer maps, lazy rollback +snapshots and dirty tracking. Independent owners serialize mutations using +SQLite OS-lock transactions plus revision checks; process death releases locks. +Existing SQLite catalog transactions are unchanged. Immutable records publish +from flushed staging files without overwrite. Missing payloads fail closed; +startup resumes tombstone cleanup and collects abandoned staging/revisions. +Older canonical catalogs, job maps and journals require explicit reset/reimport, +not silent migration or reset. Pointer maps still scale with retained jobs; +startup and journal reconstruction are not constant-time. + +Authoring workers use bounded IPC notification waits, not one-second polling. +Notifications are hints: reconnect and wake reconcile durable status and claims. +Observe-only/disconnected workers cannot acquire a lease. Cancellation registers +the wait before reading status, avoiding missed completions; empty/already-leased +scans do not self-notify. A failed connection uses a bounded backoff. None of +these storage changes grants execution permission or bypasses live dispatch +authorization. + +Learning loads and verifies the physical recorded trace in one evidence pass, +including private result sidecars and before/after source integrity checks. It +does not perform a redundant preliminary trace/evidence load or skip physical +verification on a cache hit. + The optional TypeAgent action adapter runs in the same host. It swaps the recorded protocol scopes and IDs for stable schema and action dependencies plus concrete parameters. At run time it resolves fresh contracts and scopes, and it keeps the diff --git a/ts/packages/copilot-macros/src/contracts.ts b/ts/packages/copilot-macros/src/contracts.ts index b39bb6107c..8e2c9ab5b9 100644 --- a/ts/packages/copilot-macros/src/contracts.ts +++ b/ts/packages/copilot-macros/src/contracts.ts @@ -83,6 +83,9 @@ export interface MacroAuthoringClaim { cwd: string; sessionId: string; workerId: string; + /** Durable reconciliation happens before and after this bounded IPC wait. */ + waitMs?: number; + observeOnly?: boolean; } export interface MacroAuthoringRequest { diff --git a/ts/packages/copilot-macros/src/macroChildAuthoring.ts b/ts/packages/copilot-macros/src/macroChildAuthoring.ts index 202bb6bae8..67a7695892 100644 --- a/ts/packages/copilot-macros/src/macroChildAuthoring.ts +++ b/ts/packages/copilot-macros/src/macroChildAuthoring.ts @@ -2,8 +2,8 @@ // Licensed under the MIT License. import { randomUUID } from "node:crypto"; -import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; import path from "node:path"; +import { RevisionRecords, withRecordLock } from "./revisionRecords.js"; import type { MacroAuthoringClaim, MacroAuthoringRequest, @@ -57,10 +57,7 @@ function normalizeCwd(cwd: string): string { return path.resolve(cwd).toLowerCase(); } -function parseRequests(text: string): StoredRequest[] { - if (Buffer.byteLength(text) > maxStateBytes) - throw new Error("Child authoring storage exceeds its limit."); - const value: unknown = JSON.parse(text); +function parseRequests(value: unknown): StoredRequest[] { if (!Array.isArray(value) || value.length > 512) throw new Error("Malformed child authoring storage."); for (const item of value) { @@ -118,54 +115,82 @@ export class MacroChildAuthoring { private loading?: Promise; private mutation: Promise = Promise.resolve(); private readonly file: string; + private readonly records: RevisionRecords; + private changedRequest: string | undefined; constructor( - private readonly directory: string, + directory: string, private readonly profileId = "typeagent", + private readonly notify: () => void = () => {}, ) { this.contexts = new DurableRecipeAuthoringContextStore(directory); this.file = path.join(directory, "child-authoring.json"); + this.records = new RevisionRecords(this.file); } private async load(): Promise { this.loading ??= (async () => { - try { - this.requests = parseRequests( - await readFile(this.file, "utf8"), - ); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") - throw error; - this.requests = []; - } + this.requests = parseRequests([ + ...(await this.records.load()).values(), + ]); + await this.records.purgeUnreferenced(); })(); await this.loading; } - private async update(operation: () => T): Promise { + private async update( + operation: () => T, + changedId?: string, + ): Promise { await this.load(); - const work = this.mutation.then(async () => { - const previous = structuredClone(this.requests!); - try { - const result = operation(); - const serialized = JSON.stringify(this.requests); - if ( - Buffer.byteLength(serialized) > maxStateBytes || - this.requests!.length > 512 - ) - throw new Error( - "Child authoring storage capacity exceeded.", - ); - await mkdir(this.directory, { recursive: true }); - const temporary = `${this.file}.${randomUUID()}.tmp`; - await writeFile(temporary, serialized, { flag: "wx" }); - await rename(temporary, this.file); - return result; - } catch (error) { - this.requests = previous; - throw error; - } - }); + const work = this.mutation.then(() => + withRecordLock(`${this.file}.owner`, async () => { + this.requests = parseRequests([ + ...(await this.records.load()).values(), + ]); + const previous = this.requests!.map((request) => ({ + ...request, + })); + try { + this.changedRequest = changedId; + const result = operation(); + if (this.requests!.length > 512) + throw new Error( + "Child authoring storage capacity exceeded.", + ); + const changes = new Map(); + for (const request of this.requests!) + if ( + request.requestId === this.changedRequest || + !previous.some( + (item) => item.requestId === request.requestId, + ) + ) { + if ( + Buffer.byteLength(JSON.stringify(request)) > + maxStateBytes + ) + throw new Error( + "Child authoring storage capacity exceeded.", + ); + changes.set(request.requestId, request); + } + for (const request of previous) + if ( + !this.requests!.some( + (item) => item.requestId === request.requestId, + ) + ) + changes.set(request.requestId, undefined); + await this.records.commit(changes); + if (changes.size) this.notify(); + return result; + } catch (error) { + this.requests = previous; + throw error; + } + }), + ); this.mutation = work.then( () => {}, () => {}, @@ -220,7 +245,7 @@ export class MacroChildAuthoring { this.requests!.push(request); } return structuredClone(request); - }); + }, requestId); signal.throwIfAborted(); if (!request.outcome) throw new MacroAuthoringPending(); if (request.outcome.status === "failed") @@ -244,10 +269,10 @@ export class MacroChildAuthoring { (!item.lease || Date.parse(item.lease.expiresAt) <= Date.now()); await this.load(); await this.mutation; - if (!this.requests!.some(eligible)) return undefined; const request = await this.update(() => { const request = this.requests!.find(eligible); if (!request) return undefined; + this.changedRequest = request.requestId; request.lease = { id: randomUUID(), workerId: claim.workerId, @@ -312,7 +337,7 @@ export class MacroChildAuthoring { request.outcome = structuredClone(submission.outcome); } return request.jobId; - }); + }, submission.requestId); } async forget(jobId: string): Promise { @@ -321,5 +346,9 @@ export class MacroChildAuthoring { (item) => item.jobId !== jobId, ); }); + await withRecordLock(`${this.file}.owner`, async () => { + await this.records.load(); + await this.records.purgeUnreferenced(); + }); } } diff --git a/ts/packages/copilot-macros/src/macroLearning.ts b/ts/packages/copilot-macros/src/macroLearning.ts index 255eee26b3..3715229c24 100644 --- a/ts/packages/copilot-macros/src/macroLearning.ts +++ b/ts/packages/copilot-macros/src/macroLearning.ts @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { createHash, randomUUID } from "node:crypto"; -import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import path from "node:path"; +import { RevisionRecords } from "./revisionRecords.js"; import type { MacroAuthoringClaim, MacroAuthoringRequest, @@ -148,6 +148,10 @@ export class MacroLearningEngine { private stopped = false; private readonly authoringWakeups = new Set(); private readonly childAuthoring: MacroChildAuthoring; + private readonly records: RevisionRecords; + private changedJobs: Set | undefined; + private jobSnapshots: Map | undefined; + private readonly changeWaiters = new Set<() => void>(); constructor( private readonly rootDir: string, @@ -158,7 +162,9 @@ export class MacroLearningEngine { this.childAuthoring = new MacroChildAuthoring( rootDir, authoringProfileId, + () => this.notifyAuthoring(), ); + this.records = new RevisionRecords(this.statePath()); } async configure(runtime: MacroLearningRuntime): Promise { @@ -187,6 +193,7 @@ export class MacroLearningEngine { job.draft && job.error?.includes("catalog was saved")) ) { + this.markChanged(job); job.status = "queued"; } } @@ -196,6 +203,7 @@ export class MacroLearningEngine { async close(): Promise { this.stopped = true; + this.notifyAuthoring(); this.controller?.abort(new Error("The learning owner closed.")); await this.worker; await this.mutation; @@ -265,6 +273,7 @@ export class MacroLearningEngine { activeStatuses.has(job.status) && job.macro?.state !== "approved" ) { + this.markChanged(job); job.mode = preference.mode; job.preferenceRevision = preference.revision; if (preference.mode === "off") { @@ -407,7 +416,41 @@ export class MacroLearningEngine { claim: MacroAuthoringClaim, ): Promise { await this.load(); - return this.childAuthoring.claim(claim, this.activeAuthorJobs()); + if (this.stopped) return undefined; + if (claim.waitMs !== undefined && typeof claim.waitMs !== "number") + throw new Error("Invalid authoring notification wait."); + const waitMs = Math.min(Math.max(claim.waitMs ?? 0, 0), 30_000); + if (!Number.isFinite(waitMs)) + throw new Error("Invalid authoring notification wait."); + if (this.changeWaiters.size >= 128) + throw new Error("Authoring notification wait capacity exceeded."); + let wake!: () => void; + const changed = new Promise((resolve) => { + wake = resolve; + }); + this.changeWaiters.add(wake); + const timer = setTimeout(wake, waitMs); + timer.unref(); + try { + if (!claim.observeOnly) { + const request = await this.childAuthoring.claim( + claim, + this.activeAuthorJobs(), + ); + if (request || !waitMs) return request; + } + if (!this.stopped) await changed; + // A disconnected waiter must not acquire a lease after it wakes. + // The connected host issues a fresh durable claim. + return undefined; + } finally { + clearTimeout(timer); + this.changeWaiters.delete(wake); + } + } + + private notifyAuthoring(): void { + for (const wake of this.changeWaiters) wake(); } async submitAuthoring(submission: MacroAuthoringSubmission): Promise { @@ -463,6 +506,7 @@ export class MacroLearningEngine { } for (const related of this.state!.jobs) { if (related.jobId === macro.macroId) { + this.markChanged(related); if (!this.state!.suppressed.includes(related.fingerprint)) { this.state!.suppressed.push(related.fingerprint); } @@ -1004,9 +1048,17 @@ export class MacroLearningEngine { private findJob(jobId: string): StoredJob { const job = this.state!.jobs.find((item) => item.jobId === jobId); if (!job) throw new Error(`Macro learning job not found: ${jobId}`); + this.markChanged(job); return job; } + private markChanged(job: StoredJob): void { + if (!this.changedJobs || !this.jobSnapshots) return; + this.changedJobs.add(job.jobId); + if (!this.jobSnapshots.has(job.jobId)) + this.jobSnapshots.set(job.jobId, structuredClone(job)); + } + private publicJob(job: StoredJob): MacroLearningJob { const { recipe: _recipe, @@ -1024,83 +1076,86 @@ export class MacroLearningEngine { private async load(): Promise { this.loading ??= (async () => { - try { - if ((await stat(this.statePath())).size > 16 * 1024 * 1024) { - throw new Error( - "Durable learning storage exceeds its 16 MiB limit.", - ); - } - this.state = JSON.parse( - await readFile(this.statePath(), "utf8"), - ) as LearningState; - this.state.suppressedMacroIds ??= []; - if ( - this.state.schemaVersion !== 1 || - !Array.isArray(this.state.jobs) || - !Array.isArray(this.state.preferences) || - !Array.isArray(this.state.suppressed) || - !Array.isArray(this.state.suppressedMacroIds) || - this.state.jobs.length > 1_000 || - this.state.preferences.length > 1_000 || - this.state.preferences.some( - (item) => - !modes.includes(item.mode) || - !Number.isSafeInteger(item.revision) || - item.revision < 0 || - typeof item.cwd !== "string", - ) || - this.state.jobs.some( - (job) => - typeof job.jobId !== "string" || - !/^[a-f0-9]{64}$/.test(job.jobId) || - !modes.includes(job.mode) || - ![ - "queued", - "extracting", - "building", - "awaitingAuthor", - "needsReview", - "ready", - "failed", - "cancelled", - ].includes(job.status) || - typeof job.traceId !== "string" || - typeof job.cwd !== "string" || - typeof job.fingerprint !== "string" || - (job.authoring !== undefined && - !validAuthoringIdentity(job)) || - (job.verifierAnswers !== undefined && - !validVerifierCache(job.verifierAnswers)) || - !job.attempts, - ) - ) { - throw new Error("Invalid durable learning state."); - } + const records = await this.records.load(); + await this.records.purgeUnreferenced(); + const metadata = records.get("$state") as + | Omit + | undefined; + if (records.size && !metadata) + throw new Error( + "Invalid durable learning publication: metadata is missing.", + ); + this.state = metadata + ? { + ...metadata, + jobs: [...records.entries()] + .filter(([id]) => id !== "$state") + .map(([, job]) => job as StoredJob), + } + : { + schemaVersion: 1, + preferences: [], + jobs: [], + suppressed: [], + suppressedMacroIds: [], + }; + this.state.suppressedMacroIds ??= []; + if ( + this.state.schemaVersion !== 1 || + !Array.isArray(this.state.jobs) || + !Array.isArray(this.state.preferences) || + !Array.isArray(this.state.suppressed) || + !Array.isArray(this.state.suppressedMacroIds) || + this.state.jobs.length > 1_000 || + this.state.preferences.length > 1_000 || + this.state.preferences.some( + (item) => + !modes.includes(item.mode) || + !Number.isSafeInteger(item.revision) || + item.revision < 0 || + typeof item.cwd !== "string", + ) || + this.state.jobs.some( + (job) => + typeof job.jobId !== "string" || + !/^[a-f0-9]{64}$/.test(job.jobId) || + !modes.includes(job.mode) || + ![ + "queued", + "extracting", + "building", + "awaitingAuthor", + "needsReview", + "ready", + "failed", + "cancelled", + ].includes(job.status) || + typeof job.traceId !== "string" || + typeof job.cwd !== "string" || + typeof job.fingerprint !== "string" || + (job.authoring !== undefined && + !validAuthoringIdentity(job)) || + (job.verifierAnswers !== undefined && + !validVerifierCache(job.verifierAnswers)) || + !job.attempts, + ) + ) { + throw new Error("Invalid durable learning state."); + } - function validAuthoringIdentity(job: StoredJob): boolean { - const identity = job.authoring; - return ( - !!identity && - identity.captureId === job.traceId && - /^[a-f0-9]{64}$/.test(identity.contextDigest) && - [identity.taskId, identity.tokenId].every( - (value) => - typeof value === "string" && - value.length > 0 && - value.length <= 4096, - ) - ); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") - throw error; - this.state = { - schemaVersion: 1, - preferences: [], - jobs: [], - suppressed: [], - suppressedMacroIds: [], - }; + function validAuthoringIdentity(job: StoredJob): boolean { + const identity = job.authoring; + return ( + !!identity && + identity.captureId === job.traceId && + /^[a-f0-9]{64}$/.test(identity.contextDigest) && + [identity.taskId, identity.tokenId].every( + (value) => + typeof value === "string" && + value.length > 0 && + value.length <= 4096, + ) + ); } })(); await this.loading; @@ -1108,30 +1163,35 @@ export class MacroLearningEngine { private update(operation: () => void): Promise { const result = this.mutation.then(async () => { - const before = structuredClone(this.state!); + const { jobs, ...metadataBefore } = this.state!; + const before: LearningState = { + ...structuredClone(metadataBefore), + jobs: jobs.slice(), + }; const references = new Map( this.state!.jobs.map((job) => [job.jobId, job]), ); try { + this.changedJobs = new Set(); + this.jobSnapshots = new Map(); + if (this.activeJobId) this.findJob(this.activeJobId); operation(); - const destination = this.statePath(); - await mkdir(path.dirname(destination), { recursive: true }); - const temporary = `${destination}.${randomUUID()}.tmp`; - const serialized = JSON.stringify(this.state); - if (Buffer.byteLength(serialized) > 16 * 1024 * 1024) { - throw new Error( - "Durable learning storage exceeds its 16 MiB limit.", - ); - } - await writeFile(temporary, serialized, { - encoding: "utf8", - flag: "wx", - }); - await rename(temporary, destination); + const { jobs, ...metadata } = this.state!; + const changes = new Map([ + ["$state", metadata], + ]); + for (const job of jobs) + if ( + this.changedJobs.has(job.jobId) || + !references.has(job.jobId) + ) + changes.set(job.jobId, job); + await this.records.commit(changes); + this.notifyAuthoring(); } catch (error) { - before.jobs = before.jobs.map((snapshot) => { - const original = references.get(snapshot.jobId); - if (!original) return snapshot; + before.jobs = before.jobs.map((original) => { + const snapshot = this.jobSnapshots!.get(original.jobId); + if (!snapshot) return original; for (const key of Object.keys(original)) { Reflect.deleteProperty(original, key); } @@ -1139,6 +1199,9 @@ export class MacroLearningEngine { }); this.state = before; throw error; + } finally { + this.changedJobs = undefined; + this.jobSnapshots = undefined; } }); this.mutation = result.then( diff --git a/ts/packages/copilot-macros/src/macroManager.ts b/ts/packages/copilot-macros/src/macroManager.ts index 43ec5a77f3..a8fcd9062e 100644 --- a/ts/packages/copilot-macros/src/macroManager.ts +++ b/ts/packages/copilot-macros/src/macroManager.ts @@ -9,7 +9,6 @@ import { readdir, rename, rm, - stat, writeFile, } from "node:fs/promises"; import path from "node:path"; @@ -58,6 +57,11 @@ import type { JsonValue } from "./recipeContracts.js"; import { redactTraceValue } from "./redaction.js"; import { PrivateResultStore } from "./resultEvidence.js"; import { assertMacroCapabilities } from "./resultPath.js"; +import { + publishImmutable, + publishJson, + withRecordLock, +} from "./revisionRecords.js"; const defaultRecordingTtlMs = 10 * 60 * 1000; const maxPersistedRunValueBytes = 256 * 1024; @@ -436,9 +440,7 @@ export class MacroManager { } return this.mutateCatalog(async () => { if (macro.learning) await this.learning.assertApproval(macro); - const latest = (await this.readCatalog()).find( - (item) => item.macroId === macro.macroId, - ); + const latest = await this.readSummary(macro.macroId); const existing = await this.readVersionIfPresent( macro.macroId, macro.version, @@ -639,9 +641,7 @@ export class MacroManager { ); } - const latest = (await this.readCatalog()).find( - (item) => item.macroId === macro.macroId, - ); + const latest = await this.readSummary(macro.macroId); if ( latest && (latest.version > macro.version || latest.state === "disabled") @@ -682,23 +682,32 @@ export class MacroManager { async deleteMacro(request: DeleteMacroRequest): Promise { await this.mutateCatalog(async () => { this.validateMacroId(request.macroId); - const summary = (await this.readCatalog()).find( - (item) => item.macroId === request.macroId, - ); + const summary = await this.readSummary(request.macroId); const current = summary ? await this.inspectMacro(summary) : await this.readVersionIfPresent(request.macroId, 1); if (current) await this.learning.suppress(current); + await publishJson(this.summaryPath(request.macroId), { + schemaVersion: 2, + summary: null, + }); if (!current?.learning) { await rm(path.join(this.rootDir, "macros", request.macroId), { recursive: true, force: true, }); } - const summaries = (await this.readCatalog()).filter( - (macro) => macro.macroId !== request.macroId, + const directory = path.dirname(this.summaryPath(request.macroId)); + const staged = (await readdir(directory)).filter( + (file) => + file.startsWith(`${request.macroId}.json.`) && + file.endsWith(".tmp"), + ); + await Promise.all( + staged.map((file) => + rm(path.join(directory, file), { force: true }), + ), ); - await this.writeJsonAtomic(this.catalogPath(), summaries); }); } async runMacro( @@ -1010,39 +1019,77 @@ export class MacroManager { traceId: string, ): Promise { this.validateMacroId(traceId); - const destination = path.join( - this.rootDir, - "traces", - `${traceId}.json`, - ); - if ((await stat(destination)).size > 1024 * 1024) { - throw new Error( - "Recorded learning trace exceeds its 1 MiB storage limit.", - ); - } - const trace = await this.readTrace(traceId); return new PrivateResultStore(this.rootDir).verifyLearningTrace( traceId, - trace, ); } private async readCatalog(): Promise { + await this.assertCatalogFormat(); + const directory = path.join(this.rootDir, "catalog"); + const files = await readdir(directory).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }, + ); + const summaries = await Promise.all( + files + .filter((file) => /^[a-zA-Z0-9-]+\.json$/.test(file)) + .map((file) => this.readSummary(file.slice(0, -5), false)), + ); + return summaries.filter( + (summary): summary is MacroSummary => summary !== undefined, + ); + } + + private async readSummary( + macroId: string, + checkFormat = true, + ): Promise { + if (checkFormat) await this.assertCatalogFormat(); + let pointer: { schemaVersion: number; summary: MacroSummary | null }; try { - return JSON.parse( - await readFile(this.catalogPath(), "utf8"), - ) as MacroSummary[]; + pointer = JSON.parse( + await readFile(this.summaryPath(macroId), "utf8"), + ); } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return undefined; throw error; } + if ( + pointer?.schemaVersion !== 2 || + !("summary" in pointer) || + (pointer.summary !== null && + (pointer.summary.macroId !== macroId || + !Number.isSafeInteger(pointer.summary.version) || + pointer.summary.version < 1 || + !["draft", "approved", "disabled"].includes( + pointer.summary.state, + ))) + ) + throw new Error( + "Incompatible macro catalog publication; explicitly reset/reimport the development store.", + ); + return pointer.summary ?? undefined; + } + + private async assertCatalogFormat(): Promise { + // The former array is canonical, not a derived cache. + try { + await readFile(this.catalogPath(), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return; + } + throw new Error( + "Incompatible canonical macro catalog index.json; explicitly reset/reimport the development store. It was not deleted.", + ); } private async upsertSummary(macro: CopilotToolMacro): Promise { - const summaries = (await this.readCatalog()).filter( - (summary) => summary.macroId !== macro.macroId, - ); - summaries.push({ + const summary: MacroSummary = { macroId: macro.macroId, version: macro.version, name: macro.name, @@ -1051,8 +1098,11 @@ export class MacroManager { executionClass: macro.executionClass, stepCount: macro.steps.length, updatedAt: macro.createdAt, + }; + await publishJson(this.summaryPath(macro.macroId), { + schemaVersion: 2, + summary, }); - await this.writeJsonAtomic(this.catalogPath(), summaries); } private async nextVersion(macroId: string): Promise { @@ -1074,11 +1124,7 @@ export class MacroManager { private async writeVersion(macro: CopilotToolMacro): Promise { const destination = this.versionPath(macro.macroId, macro.version); - await mkdir(path.dirname(destination), { recursive: true }); - await writeFile(destination, JSON.stringify(macro, undefined, 2), { - encoding: "utf8", - flag: "wx", - }); + await publishImmutable(destination, JSON.stringify(macro)); } private async readVersionIfPresent( @@ -1097,19 +1143,6 @@ export class MacroManager { } } - private async writeJsonAtomic( - destination: string, - value: unknown, - ): Promise { - await mkdir(path.dirname(destination), { recursive: true }); - const temporary = `${destination}.${randomUUID()}.tmp`; - await writeFile(temporary, JSON.stringify(value, undefined, 2), { - encoding: "utf8", - flag: "wx", - }); - await rename(temporary, destination); - } - private async readJson( filePath: string, notFoundMessage: string, @@ -1126,9 +1159,7 @@ export class MacroManager { private async getLatestSummary(macroId: string): Promise { this.validateMacroId(macroId); - const summary = (await this.readCatalog()).find( - (macro) => macro.macroId === macroId, - ); + const summary = await this.readSummary(macroId); if (!summary) throw new Error(`Macro not found: ${macroId}`); return summary; } @@ -1137,6 +1168,10 @@ export class MacroManager { return path.join(this.rootDir, "index.json"); } + private summaryPath(macroId: string): string { + return path.join(this.rootDir, "catalog", `${macroId}.json`); + } + private versionPath(macroId: string, version: number): string { return path.join( this.rootDir, @@ -1226,11 +1261,7 @@ export class MacroManager { : { result: redactTraceValue(boundedRun.result) }), }) as MacroRunRecord; const destination = this.runPath(run.runId); - await mkdir(path.dirname(destination), { recursive: true }); - await writeFile(destination, JSON.stringify(sanitized, undefined, 2), { - encoding: "utf8", - flag: "wx", - }); + await publishImmutable(destination, JSON.stringify(sanitized)); return sanitized; } @@ -1255,7 +1286,9 @@ export class MacroManager { } private serializeCatalog(operation: () => Promise): Promise { - const result = this.catalogMutation.then(operation, operation); + const locked = () => + withRecordLock(path.join(this.rootDir, "catalog-owner"), operation); + const result = this.catalogMutation.then(locked, locked); this.catalogMutation = result.then( () => undefined, () => undefined, diff --git a/ts/packages/copilot-macros/src/macroProgramAuthoring.ts b/ts/packages/copilot-macros/src/macroProgramAuthoring.ts index d35c615388..a008603768 100644 --- a/ts/packages/copilot-macros/src/macroProgramAuthoring.ts +++ b/ts/packages/copilot-macros/src/macroProgramAuthoring.ts @@ -23,7 +23,7 @@ import { type RecipeSpecification, type RuntimeJsonSchema, } from "./recipeCore.js"; -import { executeRecipeProgram } from "./recipeExecutor.js"; +import { RecipeExecutor } from "./recipeExecutor.js"; import { assertRecipeSemanticRegistration } from "./recipeSemantic.js"; import { isDeepStrictEqual } from "node:util"; import { z } from "zod"; @@ -552,28 +552,33 @@ async function validateFixtures( plan: MacroProgramFixturePlan, signal: AbortSignal, ): Promise { - for (const fixture of plan.fixtures) { - signal.throwIfAborted(); - const stub = fixtureHost(context.capabilities, fixture); - const result = await executeRecipeProgram( - artifact, - JSON.parse(JSON.stringify(fixture.input)) as JsonValue, - { - ...fixtureContext(context, fixture, signal), - host: stub.host, - }, - ); - if (result.status !== "completed" || result.output === undefined) - throw new Error( - `Fixture ${fixture.id}: execution ${result.status}: ${result.message ?? ""}`, - ); - const output = result.output; - stub.assertConsumed(); - assertValue(plan.outputSchema, output); - if (!sameJson(output, fixture.expectedOutput)) - throw new Error( - `Fixture ${fixture.id}: output did not meet the retained specification.`, + const executor = new RecipeExecutor({ maxWorkers: 1 }); + try { + for (const fixture of plan.fixtures) { + signal.throwIfAborted(); + const stub = fixtureHost(context.capabilities, fixture); + const result = await executor.execute( + artifact, + JSON.parse(JSON.stringify(fixture.input)) as JsonValue, + { + ...fixtureContext(context, fixture, signal), + host: stub.host, + }, ); + if (result.status !== "completed" || result.output === undefined) + throw new Error( + `Fixture ${fixture.id}: execution ${result.status}: ${result.message ?? ""}`, + ); + const output = result.output; + stub.assertConsumed(); + assertValue(plan.outputSchema, output); + if (!sameJson(output, fixture.expectedOutput)) + throw new Error( + `Fixture ${fixture.id}: output did not meet the retained specification.`, + ); + } + } finally { + await executor.close(); } const { revision: _revision, ...content } = artifact; return sealRecipeArtifact({ diff --git a/ts/packages/copilot-macros/src/macroProgramRun.ts b/ts/packages/copilot-macros/src/macroProgramRun.ts index 53bf4f1e94..8c7aa8b82d 100644 --- a/ts/packages/copilot-macros/src/macroProgramRun.ts +++ b/ts/packages/copilot-macros/src/macroProgramRun.ts @@ -1,8 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { randomUUID } from "node:crypto"; -import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { mkdir, open, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { CopilotToolMacro, @@ -25,27 +24,31 @@ export interface MacroProgramJournal { checkpoints: RecipeEffectCheckpoint[]; } -async function saveJournal( +async function appendCheckpoint( destination: string, - journal: MacroProgramJournal, - create: boolean, + checkpoint: RecipeEffectCheckpoint, ) { - await mkdir(path.dirname(destination), { recursive: true }); - if (create) { - await writeFile(destination, JSON.stringify(journal), { - encoding: "utf8", - flag: "wx", - flush: true, - }); - return; + // Never recreate a missing journal or replace the exclusive run claim. + const handle = await open(destination, "r+"); + try { + const bytes = Buffer.from(`${JSON.stringify({ checkpoint })}\n`); + const size = (await handle.stat()).size; + let written = 0; + while (written < bytes.length) { + const result = await handle.write( + bytes, + written, + bytes.length - written, + size + written, + ); + if (!result.bytesWritten) + throw new Error("Program journal append made no progress."); + written += result.bytesWritten; + } + await handle.sync(); + } finally { + await handle.close(); } - const temporary = `${destination}.${randomUUID()}.tmp`; - await writeFile(temporary, JSON.stringify(journal), { - encoding: "utf8", - flag: "wx", - flush: true, - }); - await rename(temporary, destination); } export function programJournalPath(rootDir: string, runId: string) { @@ -53,9 +56,41 @@ export function programJournalPath(rootDir: string, runId: string) { } export async function readProgramJournal(rootDir: string, runId: string) { - return JSON.parse( - await readFile(programJournalPath(rootDir, runId), "utf8"), - ) as MacroProgramJournal; + const lines = ( + await readFile(programJournalPath(rootDir, runId), "utf8") + ).split("\n"); + const header = JSON.parse(lines.shift()!); + if ( + header.schemaVersion !== 2 || + header.runId !== runId || + typeof header.macroId !== "string" || + !Number.isSafeInteger(header.version) || + typeof header.startedAt !== "string" + ) + throw new Error( + "Incompatible canonical program journal; explicitly reset/reimport the development store. It was not deleted.", + ); + const journal: MacroProgramJournal = { ...header, checkpoints: [] }; + const checkpoints = new Map(); + // An interrupted final append is not a durable acknowledgement. + lines.pop(); + for (const line of lines) { + const { checkpoint } = JSON.parse(line); + if ( + !checkpoint || + typeof checkpoint.callId !== "string" || + ![ + "not_dispatched", + "dispatched", + "completed", + "uncertain", + ].includes(checkpoint.phase) + ) + throw new Error("Invalid durable program checkpoint."); + checkpoints.set(checkpoint.callId, checkpoint); + } + journal.checkpoints = [...checkpoints.values()]; + return journal; } export async function runMacroProgram( @@ -82,8 +117,18 @@ export async function runMacroProgram( checkpoints: [], }; const destination = programJournalPath(rootDir, request.runId); - await saveJournal(destination, journal, true); + await mkdir(path.dirname(destination), { recursive: true }); + await writeFile( + destination, + `${JSON.stringify({ ...journal, schemaVersion: 2 })}\n`, + { + encoding: "utf8", + flag: "wx", + flush: true, + }, + ); let checkpointMutation = Promise.resolve(); + const checkpointIndexes = new Map(); let result: RecipeRunResult; try { result = await runtime.execute(artifact, input as JsonValue, { @@ -95,24 +140,26 @@ export async function runMacroProgram( 60_000, ), onCheckpoint: (checkpoint) => { + const saved = structuredClone(checkpoint); checkpointMutation = checkpointMutation.then(async () => { - if (checkpoint.phase === "dispatched") { + if (saved.phase === "dispatched") { signal.throwIfAborted(); await assertCurrent(); } - const index = journal.checkpoints.findIndex( - (value) => value.callId === checkpoint.callId, - ); - if (index === -1) - journal.checkpoints.push(structuredClone(checkpoint)); - else - journal.checkpoints[index] = - structuredClone(checkpoint); - await saveJournal(destination, journal, false); + await appendCheckpoint(destination, saved); + const index = checkpointIndexes.get(saved.callId); + if (index === undefined) { + checkpointIndexes.set( + saved.callId, + journal.checkpoints.length, + ); + journal.checkpoints.push(saved); + } else journal.checkpoints[index] = saved; }); return checkpointMutation; }, }); + await checkpointMutation; if ( !result.accepted || result.runId !== request.runId || diff --git a/ts/packages/copilot-macros/src/recipeContracts.ts b/ts/packages/copilot-macros/src/recipeContracts.ts index e6ef58ce03..b4fd1d3753 100644 --- a/ts/packages/copilot-macros/src/recipeContracts.ts +++ b/ts/packages/copilot-macros/src/recipeContracts.ts @@ -421,6 +421,10 @@ export interface RecipeRunResult extends RecipeRevisionRef { checkpoints: RecipeEffectCheckpoint[]; completedCallIds: string[]; semanticEvidence: RecipeSemanticEvidence[]; + performance?: { + workerQueueMs: number; + workerReused: boolean; + }; fallback: "never"; } diff --git a/ts/packages/copilot-macros/src/recipeExecutor.ts b/ts/packages/copilot-macros/src/recipeExecutor.ts index 5b0fc3360c..ad0dcb1173 100644 --- a/ts/packages/copilot-macros/src/recipeExecutor.ts +++ b/ts/packages/copilot-macros/src/recipeExecutor.ts @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { Worker } from "node:worker_threads"; +import type { Worker } from "node:worker_threads"; +import { + configureRecipeWorkerAssets, + RecipeWorkerPool, +} from "./recipeWorkerPool.js"; import { verifyRecipeExecutable } from "./recipeCompiler.js"; import type { JsonValue, @@ -18,17 +22,66 @@ import { validateRecipeValue, } from "./recipeValidation.js"; -let executorAssets: { workerUrl: URL; quickjsModulePath?: string } | undefined; - /** Set relocatable worker assets when packaging without node_modules. */ export function configureRecipeExecutorAssets(assets: { workerUrl: URL; quickjsModulePath?: string; }): void { - executorAssets = { - ...assets, - workerUrl: new URL(assets.workerUrl.href), - }; + configureRecipeWorkerAssets(assets); +} + +export class RecipeExecutor { + private readonly pool: RecipeWorkerPool; + private readonly controller = new AbortController(); + private readonly executions = new Set>(); + private closed = false; + private closing?: Promise; + + constructor(options: { maxWorkers?: number } = {}) { + this.pool = new RecipeWorkerPool(options.maxWorkers); + } + + execute( + artifact: RecipeArtifact, + input: JsonValue, + context: RecipeExecutionContext, + options: RecipeExecutorOptions = {}, + ): Promise { + if (this.closed) + return Promise.reject(new Error("Recipe executor is closed.")); + const execution = executeWithPool( + artifact, + input, + { + ...context, + signal: AbortSignal.any([ + context.signal, + this.controller.signal, + ]), + }, + options, + this.pool, + ); + this.executions.add(execution); + const finished = () => { + this.executions.delete(execution); + }; + void execution.then(finished, finished); + return execution; + } + + close(): Promise { + this.closed = true; + this.closing ??= (async () => { + this.controller.abort(); + try { + await this.pool.close(); + } finally { + await Promise.allSettled([...this.executions]); + } + })(); + return this.closing; + } } export interface RecipeExecutorOptions { @@ -82,17 +135,39 @@ async function preflightCapabilities( context: RecipeExecutionContext, signal: AbortSignal, ) { - for (const capability of artifact.capabilities) { - const checked = await observe( - () => context.host.inspectCapability(capability, context.binding), - signal, + const concurrency = 4; + for ( + let offset = 0; + offset < artifact.capabilities.length; + offset += concurrency + ) { + const capabilities = artifact.capabilities.slice( + offset, + offset + concurrency, ); - if (checked.status !== "available") return checked; - if (checkRecipeCapability(capability, checked.capability).length) - return { - status: "unsupported" as const, - message: "Capability contract changed during preflight.", - }; + const results = await Promise.all( + capabilities.map((capability) => + observe( + () => + context.host.inspectCapability( + capability, + context.binding, + ), + signal, + ), + ), + ); + for (const [index, checked] of results.entries()) { + if (checked.status !== "available") return checked; + if ( + checkRecipeCapability(capabilities[index], checked.capability) + .length + ) + return { + status: "unsupported" as const, + message: "Capability contract changed during preflight.", + }; + } } return undefined; } @@ -245,11 +320,24 @@ class Broker { private readonly bounds: Bounds, private readonly options: RecipeExecutorOptions, private readonly signal: AbortSignal, + private readonly cancel: () => void, ) {} + isConcurrentRead(request: Request): boolean { + return ( + request.kind === "tools" && + this.artifact.capabilities.some( + (capability) => + capability.id === request.id && + capability.effect === "read", + ) + ); + } + private stop(status: RecipeRunResult["status"], message: string): never { this.result.status = status; this.result.message = message; + this.cancel(); throw new Stopped(message); } @@ -263,7 +351,14 @@ class Broker { ); } - async dispatch(request: Request): Promise { + dispatch(request: Request): Promise { + return this.dispatchRequest(request).catch((error: unknown) => { + this.cancel(); + throw error; + }); + } + + private async dispatchRequest(request: Request): Promise { if (this.signal.aborted) throw new Stopped("Recipe execution stopped."); if (++this.calls > this.bounds.maxCalls) this.stop("limit_exceeded", "Broker call limit exceeded."); @@ -442,21 +537,15 @@ async function runGuest( options: RecipeExecutorOptions, signal: AbortSignal, deadline: number, + pool: RecipeWorkerPool, + cancel: () => void, ): Promise { if (signal.aborted) throw new Stopped("Runtime stopped."); - const worker = new Worker( - executorAssets?.workerUrl ?? - new URL("./recipeExecutorWorker.js", import.meta.url), - { - workerData: { - source: guestSource(artifact, inputText, bounds.maxJsonBytes), - bounds, - deadline, - quickjsModulePath: executorAssets?.quickjsModulePath, - }, - resourceLimits: { maxOldGenerationSizeMb: 64, stackSizeMb: 4 }, - }, - ); + const lease = await pool.acquire(signal); + result.performance = { + workerQueueMs: lease.queueMs, + workerReused: lease.reused, + }; const broker = new Broker( artifact, context, @@ -464,15 +553,37 @@ async function runGuest( bounds, options, signal, + cancel, ); + let healthy = false; try { - const value = await workerResult(worker, broker, bounds, signal); + const value = await workerResult( + lease.worker, + broker, + bounds, + signal, + cancel, + () => + lease.worker.postMessage({ + kind: "execute", + data: { + source: guestSource( + artifact, + inputText, + bounds.maxJsonBytes, + ), + bounds, + deadline, + }, + }), + ); if (validateRecipeValue(artifact.outputSchema, value).length) throw new Error("Recipe output does not satisfy its schema."); result.output = value; result.status = "completed"; + healthy = true; } finally { - await worker.terminate(); + await lease.release(healthy); } } @@ -481,73 +592,117 @@ function workerResult( broker: Broker, bounds: Bounds, signal: AbortSignal, + cancel: () => void, + start: () => void, ): Promise { return new Promise((resolve, reject) => { let settled = false; - let busy = false; let calls = 0; + let exclusive = false; + const queued: Request[] = []; + const active = new Set(); + const clean = () => { + signal.removeEventListener("abort", abort); + worker.off("error", stop); + worker.off("exit", exited); + worker.off("message", receive); + }; const stop = (error: unknown) => { if (settled) return; settled = true; - signal.removeEventListener("abort", abort); + clean(); + cancel(); reject(error); }; const abort = () => stop(new Stopped("Runtime stopped.")); - signal.addEventListener("abort", abort, { once: true }); - if (signal.aborted) return abort(); - worker.on("error", stop); - worker.on("exit", () => - stop(new Error("Guest worker exited unexpectedly.")), - ); - worker.on("message", (message: { kind: string; text: string }) => { + const exited = () => + stop(new Error("Guest worker exited unexpectedly.")); + const pump = () => { + while ( + !settled && + !signal.aborted && + queued.length && + active.size < 4 && + !exclusive + ) { + const request = queued[0]; + const concurrent = broker.isConcurrentRead(request); + if (!concurrent && active.size) return; + queued.shift(); + exclusive = !concurrent; + active.add(request.request); + void broker.dispatch(request).then((value) => { + try { + if (settled || signal.aborted) return; + const text = jsonText( + { request: request.request, value }, + bounds.maxJsonBytes, + ); + active.delete(request.request); + exclusive = false; + worker.postMessage(text); + pump(); + } catch (error) { + stop(error); + } + }, stop); + } + }; + const receive = (message: { kind: string; text: string }) => { if (settled) return; - void (async () => { - if (Buffer.byteLength(message.text) > bounds.maxJsonBytes) + try { + if ( + !message || + typeof message.text !== "string" || + Buffer.byteLength(message.text) > bounds.maxJsonBytes + ) throw new Error("Guest JSON limit exceeded."); if (message.kind === "failure") throw new Error(message.text); if (message.kind === "result") { - if (busy) + if (active.size || queued.length) throw new Error("Guest completed with pending calls."); const value: JsonValue = JSON.parse(message.text); jsonText(value, bounds.maxJsonBytes); settled = true; - signal.removeEventListener("abort", abort); + clean(); resolve(value); return; } - if ( - message.kind !== "request" || - busy || - ++calls > bounds.maxCalls - ) + if (message.kind !== "request" || ++calls > bounds.maxCalls) throw new Error("Invalid or excessive broker requests."); - busy = true; const request = JSON.parse(message.text) as Request; if ( + !request || request.request !== calls || typeof request.id !== "string" || !["tools", "semantic"].includes(request.kind) ) throw new Error("Invalid broker request."); - const value = await broker.dispatch(request); - if (settled || signal.aborted) return; - worker.postMessage( - jsonText( - { request: request.request, value }, - bounds.maxJsonBytes, - ), - ); - busy = false; - })().catch(stop); - }); + queued.push(request); + pump(); + } catch (error) { + stop(error); + } + }; + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) return abort(); + worker.on("error", stop); + worker.on("exit", exited); + worker.on("message", receive); + try { + start(); + } catch (error) { + stop(error); + } }); } -export async function executeRecipeProgram( +async function executeWithPool( artifact: RecipeArtifact, input: JsonValue, context: RecipeExecutionContext, - options: RecipeExecutorOptions = {}, + options: RecipeExecutorOptions, + pool: RecipeWorkerPool, ): Promise { const result: RecipeRunResult = { recipeId: artifact.recipeId, @@ -599,6 +754,8 @@ export async function executeRecipeProgram( options, controller.signal, deadline, + pool, + cancel, ); } catch (error) { if (result.status === "failed") { @@ -654,3 +811,14 @@ export async function executeRecipeProgram( } return result; } + +const defaultExecutor = new RecipeExecutor(); + +export function executeRecipeProgram( + artifact: RecipeArtifact, + input: JsonValue, + context: RecipeExecutionContext, + options: RecipeExecutorOptions = {}, +): Promise { + return defaultExecutor.execute(artifact, input, context, options); +} diff --git a/ts/packages/copilot-macros/src/recipeExecutorWorker.ts b/ts/packages/copilot-macros/src/recipeExecutorWorker.ts index 0e6a9c52ec..12434a453a 100644 --- a/ts/packages/copilot-macros/src/recipeExecutorWorker.ts +++ b/ts/packages/copilot-macros/src/recipeExecutorWorker.ts @@ -8,7 +8,6 @@ import { parentPort, workerData } from "node:worker_threads"; interface WorkerInput { source: string; deadline: number; - quickjsModulePath?: string; bounds: { memoryLimitBytes: number; stackLimitBytes: number; @@ -17,6 +16,22 @@ interface WorkerInput { }; } +const port = parentPort; +let modulePromise: + | ReturnType< + typeof import("@tootallnate/quickjs-emscripten").newQuickJSWASMModule + > + | undefined; + +function getModule() { + if (modulePromise) return modulePromise; + const { newQuickJSWASMModule } = createRequire(import.meta.url)( + (workerData as { quickjsModulePath?: string }).quickjsModulePath ?? + "@tootallnate/quickjs-emscripten", + ) as typeof import("@tootallnate/quickjs-emscripten"); + return (modulePromise = newQuickJSWASMModule()); +} + function evaluate(vm: QuickJSContext, source: string) { const evaluated = vm.evalCode(source, "recipe.js"); if (evaluated.error) { @@ -27,103 +42,178 @@ function evaluate(vm: QuickJSContext, source: string) { } async function executeGuest(data: WorkerInput): Promise { - const port = parentPort!; - const { newQuickJSWASMModule } = createRequire(import.meta.url)( - data.quickjsModulePath ?? "@tootallnate/quickjs-emscripten", - ) as typeof import("@tootallnate/quickjs-emscripten"); - const module = await newQuickJSWASMModule(); + const module = await getModule(); const runtime = module.newRuntime(); - runtime.setMemoryLimit(data.bounds.memoryLimitBytes); - runtime.setMaxStackSize(data.bounds.stackLimitBytes); - runtime.setInterruptHandler(() => Date.now() >= data.deadline); - const vm = runtime.newContext(); + let vm: QuickJSContext | undefined; + let deliver: ReturnType | undefined; + let wake: (() => void) | undefined; const requests: string[] = []; + const responses: string[] = []; + const pending = new Set(); let calls = 0; let protocolError: string | undefined; let completion: | { status: string; value?: unknown; message?: string } | undefined; - const install = (name: string, callback: (text: string) => void) => { - const handle = vm.newFunction(name, (argument) => { - try { - if (!argument || vm.typeof(argument) !== "string") - throw new Error("Invalid guest protocol."); - const text = vm.getString(argument); - if (Buffer.byteLength(text) > data.bounds.maxJsonBytes) - throw new Error("Guest JSON limit exceeded."); - callback(text); - return vm.undefined; - } catch (error) { - protocolError = - error instanceof Error ? error.message : "Broker failure."; - return vm.throw( - error instanceof Error - ? error - : new Error("Broker failure."), + const receive = (text: unknown) => { + if ( + typeof text !== "string" || + Buffer.byteLength(text) > data.bounds.maxJsonBytes + ) + protocolError = "Invalid broker response."; + else responses.push(text); + wake?.(); + wake = undefined; + }; + port!.on("message", receive); + let output: string; + try { + runtime.setMemoryLimit(data.bounds.memoryLimitBytes); + runtime.setMaxStackSize(data.bounds.stackLimitBytes); + runtime.setInterruptHandler(() => Date.now() >= data.deadline); + const context = runtime.newContext(); + vm = context; + const install = (name: string, callback: (text: string) => void) => { + const handle = context.newFunction(name, (argument) => { + try { + if (!argument || context.typeof(argument) !== "string") + throw new Error("Invalid guest protocol."); + const text = context.getString(argument); + if (Buffer.byteLength(text) > data.bounds.maxJsonBytes) + throw new Error("Guest JSON limit exceeded."); + callback(text); + return context.undefined; + } catch (error) { + protocolError = + error instanceof Error + ? error.message + : "Broker failure."; + return context.throw( + error instanceof Error + ? error + : new Error("Broker failure."), + ); + } + }); + context.setProp(context.global, name, handle); + handle.dispose(); + }; + install("__recipeSend", (text) => { + if (++calls > data.bounds.maxCalls) + throw new Error("Broker call limit exceeded."); + requests.push(text); + }); + install("__recipeFinish", (text) => { + if (completion) throw new Error("Duplicate guest completion."); + completion = JSON.parse(text); + }); + deliver = evaluate(context, data.source); + while (!completion) { + if (Date.now() >= data.deadline) + throw new Error("Guest deadline exceeded."); + const jobs = runtime.executePendingJobs(100); + if (jobs.error) { + jobs.error.dispose(); + throw new Error( + "Guest job failed or exceeded resource limits.", ); } - }); - vm.setProp(vm.global, name, handle); - handle.dispose(); - }; - install("__recipeSend", (text) => { - if (++calls > data.bounds.maxCalls) - throw new Error("Broker call limit exceeded."); - requests.push(text); - }); - install("__recipeFinish", (text) => { - if (completion) throw new Error("Duplicate guest completion."); - completion = JSON.parse(text); - }); - const deliver = evaluate(vm, data.source); - while (!completion) { - if (Date.now() >= data.deadline) - throw new Error("Guest deadline exceeded."); - const jobs = runtime.executePendingJobs(100); - if (jobs.error) { - jobs.error.dispose(); - throw new Error("Guest job failed or exceeded resource limits."); + if (protocolError) throw new Error(protocolError); + if (completion) break; + for (const text of requests.splice(0)) { + const request = JSON.parse(text) as { request: number }; + if ( + !Number.isSafeInteger(request.request) || + pending.has(request.request) + ) + throw new Error("Invalid guest request identity."); + pending.add(request.request); + port!.postMessage({ kind: "request", text }); + } + if (responses.length) { + for (const text of responses.splice(0)) { + const reply = JSON.parse(text) as { request: number }; + if (!pending.delete(reply.request)) + throw new Error("Unknown broker response."); + const response = context.newString(text); + try { + context + .unwrapResult( + context.callFunction( + deliver, + context.undefined, + response, + ), + ) + .dispose(); + } finally { + response.dispose(); + } + } + continue; + } + if (!runtime.hasPendingJob()) { + if (!pending.size) + throw new Error("Guest promise did not settle."); + await new Promise((resolve) => { + wake = resolve; + }); + } } - if (protocolError) throw new Error(protocolError); - if (completion) break; - const request = requests.shift(); - if (request) { - const text = await new Promise((resolve) => { - port.once("message", resolve); - port.postMessage({ kind: "request", text: request }); - }); - const response = vm.newString(text); + if (requests.length || pending.size) + throw new Error("Guest left unawaited broker calls."); + if ( + completion?.status !== "completed" || + completion.value === undefined + ) + throw new Error(completion?.message ?? "Guest failed."); + output = JSON.stringify(completion.value); + } finally { + port!.off("message", receive); + try { + deliver?.dispose(); + } finally { try { - vm.unwrapResult( - vm.callFunction(deliver, vm.undefined, response), - ).dispose(); + vm?.dispose(); } finally { - response.dispose(); + runtime.dispose(); } - } else if (!runtime.hasPendingJob()) { - throw new Error("Guest promise did not settle."); } } - if (requests.length) throw new Error("Guest left unawaited broker calls."); - if (completion?.status !== "completed" || completion.value === undefined) - throw new Error(completion?.message ?? "Guest failed."); - const output = JSON.stringify(completion.value); - deliver.dispose(); - vm.dispose(); - runtime.dispose(); - port.postMessage({ kind: "result", text: output }); + port!.postMessage({ kind: "result", text: output }); } -if (parentPort) { - // Termination releases the complete WASM heap even when QuickJS cannot - // unwind an allocation failure. No failed VM is reused or retained. - void executeGuest(workerData as WorkerInput).catch((error: unknown) => { - parentPort!.postMessage({ - kind: "failure", - text: - error instanceof Error - ? error.message - : "Unexpected guest failure.", - }); +if (port) { + let busy = false; + port.on("message", (message: unknown) => { + if (typeof message === "string") return; + if ( + !message || + typeof message !== "object" || + !("kind" in message) || + message.kind !== "execute" || + !("data" in message) || + busy + ) { + port.postMessage({ + kind: "failure", + text: "Invalid worker invocation.", + }); + return; + } + busy = true; + void executeGuest(message.data as WorkerInput) + .catch((error: unknown) => { + port.postMessage({ + kind: "failure", + text: + error instanceof Error + ? error.message + : "Unexpected guest failure.", + }); + }) + .finally(() => { + busy = false; + }); }); } diff --git a/ts/packages/copilot-macros/src/recipeWorkerPool.ts b/ts/packages/copilot-macros/src/recipeWorkerPool.ts new file mode 100644 index 0000000000..f5fdaf0916 --- /dev/null +++ b/ts/packages/copilot-macros/src/recipeWorkerPool.ts @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Worker } from "node:worker_threads"; + +interface Assets { + workerUrl: URL; + quickjsModulePath?: string; +} + +let assets: Assets | undefined; +let generation = 0; +const pools = new Set(); + +export function configureRecipeWorkerAssets(value: Assets): void { + assets = { ...value, workerUrl: new URL(value.workerUrl.href) }; + generation++; + for (const pool of pools) pool.invalidate(); +} + +interface Entry { + worker: Worker; + generation: number; + busy: boolean; + used: boolean; + failed: boolean; + idleTimer?: ReturnType; + termination?: Promise; +} + +interface Waiter { + started: number; + signal: AbortSignal; + abort: () => void; + resolve: (lease: RecipeWorkerLease) => void; + reject: (error: Error) => void; +} + +export interface RecipeWorkerLease { + worker: Worker; + queueMs: number; + reused: boolean; + release(healthy: boolean): Promise; +} + +export class RecipeWorkerPool { + private readonly entries = new Map(); + private readonly waiters: Waiter[] = []; + private closed = false; + + constructor(private readonly capacity = 8) { + if (!Number.isSafeInteger(capacity) || capacity < 1 || capacity > 8) + throw new Error("Worker capacity must be between 1 and 8."); + pools.add(this); + } + + acquire(signal: AbortSignal): Promise { + if (this.closed) + return Promise.reject(new Error("Recipe worker owner is closed.")); + if (signal.aborted) + return Promise.reject(new Error("Worker admission cancelled.")); + if (this.waiters.length >= 64) + return Promise.reject( + new Error("Recipe worker admission queue is full."), + ); + return new Promise((resolve, reject) => { + const waiter: Waiter = { + started: performance.now(), + signal, + resolve, + reject, + abort: () => { + const index = this.waiters.indexOf(waiter); + if (index !== -1) this.waiters.splice(index, 1); + signal.removeEventListener("abort", waiter.abort); + reject(new Error("Worker admission cancelled.")); + }, + }; + signal.addEventListener("abort", waiter.abort, { once: true }); + this.waiters.push(waiter); + this.drain(); + }); + } + + private create(): Entry { + const worker = new Worker( + assets?.workerUrl ?? + new URL("./recipeExecutorWorker.js", import.meta.url), + { + workerData: { quickjsModulePath: assets?.quickjsModulePath }, + resourceLimits: { maxOldGenerationSizeMb: 64, stackSizeMb: 4 }, + }, + ); + const entry: Entry = { + worker, + generation, + busy: false, + used: false, + failed: false, + }; + this.entries.set(worker, entry); + worker.on("error", (error: Error) => { + entry.failed = true; + if (!entry.busy) console.error("Idle recipe worker failed.", error); + }); + worker.on("exit", (code) => { + if (!entry.busy && !entry.termination) + console.error("Idle recipe worker exited unexpectedly.", code); + clearTimeout(entry.idleTimer); + this.entries.delete(worker); + this.drain(); + }); + return entry; + } + + private drain(): void { + if (this.closed) return; + while (this.waiters.length) { + let entry = [...this.entries.values()].find( + (value) => + !value.busy && + !value.failed && + value.generation === generation, + ); + if (!entry && this.entries.size >= this.capacity) return; + const waiter = this.waiters.shift()!; + waiter.signal.removeEventListener("abort", waiter.abort); + try { + entry ??= this.create(); + const selected = entry; + clearTimeout(entry.idleTimer); + entry.busy = true; + entry.worker.ref(); + const reused = entry.used; + entry.used = true; + let released = false; + waiter.resolve({ + worker: entry.worker, + queueMs: performance.now() - waiter.started, + reused, + release: async (healthy) => { + if (released) return; + released = true; + if ( + !healthy || + selected.failed || + this.closed || + selected.generation !== generation + ) { + await this.retire(selected); + } else { + selected.busy = false; + selected.worker.unref(); + selected.idleTimer = setTimeout( + () => this.retireIdle(selected), + 30_000, + ); + selected.idleTimer.unref(); + this.drain(); + } + }, + }); + } catch (error) { + waiter.reject( + error instanceof Error ? error : new Error(String(error)), + ); + } + } + } + + private retire(entry: Entry): Promise { + clearTimeout(entry.idleTimer); + entry.failed = true; + entry.busy = true; + entry.termination ??= entry.worker.terminate(); + return entry.termination; + } + + private retireIdle(entry: Entry): void { + void this.retire(entry).catch((error: unknown) => { + console.error("Failed to terminate an idle recipe worker.", error); + }); + } + + invalidate(): void { + for (const entry of this.entries.values()) + if (!entry.busy) this.retireIdle(entry); + } + + async close(): Promise { + this.closed = true; + pools.delete(this); + for (const waiter of this.waiters.splice(0)) { + waiter.signal.removeEventListener("abort", waiter.abort); + waiter.reject(new Error("Recipe worker owner is closed.")); + } + await Promise.all( + [...this.entries.values()].map((entry) => this.retire(entry)), + ); + } +} diff --git a/ts/packages/copilot-macros/src/resultEvidence.ts b/ts/packages/copilot-macros/src/resultEvidence.ts index 4643935713..ccdc5b2391 100644 --- a/ts/packages/copilot-macros/src/resultEvidence.ts +++ b/ts/packages/copilot-macros/src/resultEvidence.ts @@ -321,9 +321,9 @@ export class PrivateResultStore { async verifyLearningTrace( traceId: string, - trace: RecordedInteractionTrace, + trace?: RecordedInteractionTrace, ): Promise { - verifiedLearningTraces.delete(trace); + if (trace) verifiedLearningTraces.delete(trace); if (!/^[a-zA-Z0-9-]+$/.test(traceId)) throw new ResultEvidenceError( "invalidReference", @@ -333,12 +333,30 @@ export class PrivateResultStore { const sourceTrace: RecordedInteractionTrace = JSON.parse( bytes.toString("utf8"), ); + trace ??= sourceTrace; if (JSON.stringify(sourceTrace) !== JSON.stringify(trace)) throw new ResultEvidenceError( "evidenceIntegrity", "Learning trace does not match its physical source.", ); assertLearningTraceIdentity(sourceTrace); + const hasEvidence = sourceTrace.toolCalls.some( + (call) => call.resultArtifact || call.modelResultArtifact, + ); + if ( + (sourceTrace.schemaVersion === 1 && + (hasEvidence || sourceTrace.capabilities !== undefined)) || + (sourceTrace.schemaVersion === 2 && + (!hasEvidence || + sourceTrace.capabilities?.length !== 1 || + sourceTrace.capabilities[0] !== + "private-result-evidence-v1")) || + (sourceTrace.schemaVersion !== 1 && sourceTrace.schemaVersion !== 2) + ) + throw new ResultEvidenceError( + "evidenceIntegrity", + "Unsupported recorded trace schema or capabilities.", + ); const originals: unknown[] = []; for (const call of sourceTrace.toolCalls) { let original = call.result; diff --git a/ts/packages/copilot-macros/src/revisionRecords.ts b/ts/packages/copilot-macros/src/revisionRecords.ts new file mode 100644 index 0000000000..22554f557a --- /dev/null +++ b/ts/packages/copilot-macros/src/revisionRecords.ts @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID } from "node:crypto"; +import { link, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { createRequire } from "node:module"; + +export async function withRecordLock( + file: string, + operation: () => Promise, +): Promise { + await mkdir(path.dirname(file), { recursive: true }); + // SQLite's OS locks are released on owner death, unlike a sentinel file. + const { DatabaseSync } = createRequire(import.meta.url)("node:sqlite") as { + DatabaseSync: new (file: string) => { + exec(sql: string): void; + close(): void; + }; + }; + const database = new DatabaseSync(`${file}.owner.sqlite`); + try { + for (let attempt = 0; ; attempt++) { + try { + database.exec("BEGIN IMMEDIATE"); + break; + } catch (error) { + if ( + typeof error !== "object" || + error === null || + !("message" in error) || + !String(error.message).includes("locked") || + attempt === 200 + ) + throw error; + await delay(25); + } + } + try { + const result = await operation(); + database.exec("COMMIT"); + return result; + } catch (error) { + database.exec("ROLLBACK"); + throw error; + } + } finally { + database.close(); + } +} + +export async function publishImmutable( + destination: string, + bytes: string, +): Promise { + await mkdir(path.dirname(destination), { recursive: true }); + const staging = `${destination}.${randomUUID()}.tmp`; + try { + await writeFile(staging, bytes, { flag: "wx", flush: true }); + await link(staging, destination); + } finally { + await rm(staging, { force: true }); + } +} + +interface Publication { + schemaVersion: 2; + revision: string; + records: Record; +} + +export async function publishJson(destination: string, value: unknown) { + await mkdir(path.dirname(destination), { recursive: true }); + const temporary = `${destination}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify(value), { + flag: "wx", + flush: true, + }); + await rename(temporary, destination); + } finally { + await rm(temporary, { force: true }); + } +} + +/** Payloads are immutable; only the small record-to-revision map is replaced. */ +export class RevisionRecords { + private publication?: Publication; + private readonly sizes = new Map(); + + constructor(private readonly file: string) {} + + private payloadPath(digest: string) { + return path.join(`${this.file}.records`, `${digest}.json`); + } + + async load(): Promise> { + return withRecordLock(this.file, () => this.loadLocked()); + } + + private async loadLocked(): Promise> { + let publication: Publication; + try { + publication = JSON.parse(await readFile(this.file, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + publication = { schemaVersion: 2, revision: "", records: {} }; + } + if ( + publication.schemaVersion !== 2 || + typeof publication.revision !== "string" || + !publication.records || + typeof publication.records !== "object" || + Array.isArray(publication.records) || + Object.values(publication.records).some( + (digest) => !/^[a-f0-9]{64}$/.test(digest), + ) + ) + throw new Error( + `Incompatible canonical macro store: ${this.file}. Explicitly reset/reimport this development store; it was not deleted.`, + ); + const entries = await Promise.all( + Object.entries(publication.records).map(async ([id, digest]) => { + const bytes = await readFile(this.payloadPath(digest), "utf8"); + this.sizes.set(digest, Buffer.byteLength(bytes)); + if (createHash("sha256").update(bytes).digest("hex") !== digest) + throw new Error("Immutable macro record digest mismatch."); + return [id, JSON.parse(bytes)] as const; + }), + ); + this.publication = publication; + this.assertSize(publication.records); + return new Map(entries); + } + + private assertSize(records: Record): void { + const bytes = Object.values(records).reduce( + (total, digest) => total + this.sizes.get(digest)!, + 0, + ); + if (bytes > 16 * 1024 * 1024) + throw new Error("Durable macro storage exceeds its 16 MiB limit."); + } + + async commit(changes: Map): Promise { + if (!this.publication) await this.load(); + await withRecordLock(this.file, () => this.commitLocked(changes)); + } + + private async commitLocked( + changes: Map, + ): Promise { + const previous = this.publication!; + let revision = ""; + try { + revision = JSON.parse(await readFile(this.file, "utf8")).revision; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (revision !== previous.revision) + throw new Error( + "Macro record publication revision changed; reopen the owner before retrying.", + ); + const records = { ...previous.records }; + for (const [id, value] of changes) { + if (value === undefined) { + delete records[id]; + continue; + } + const bytes = JSON.stringify(value); + if (Buffer.byteLength(bytes) > 16 * 1024 * 1024) + throw new Error( + "Durable macro record exceeds its 16 MiB limit.", + ); + const digest = createHash("sha256").update(bytes).digest("hex"); + this.sizes.set(digest, Buffer.byteLength(bytes)); + if (records[id] === digest) continue; + await mkdir(`${this.file}.records`, { recursive: true }); + try { + await publishImmutable(this.payloadPath(digest), bytes); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") + throw error; + if ( + (await readFile(this.payloadPath(digest), "utf8")) !== bytes + ) + throw new Error("Immutable macro record collision."); + } + records[id] = digest; + } + if ( + Object.keys(records).length === + Object.keys(previous.records).length && + Object.entries(records).every( + ([id, value]) => previous.records[id] === value, + ) + ) + return; + const next: Publication = { + schemaVersion: 2, + revision: randomUUID(), + records, + }; + this.assertSize(records); + await publishJson(this.file, next); + this.publication = next; + } + + /** Called only after publication, never while readers can select old payloads. */ + async purgeUnreferenced(): Promise { + await withRecordLock(this.file, async () => { + const current = new RevisionRecords(this.file); + await current.loadLocked(); + await current.purgeLocked(); + }); + } + + private async purgeLocked(): Promise { + const { readdir } = await import("node:fs/promises"); + const retained = new Set(Object.values(this.publication!.records)); + const directory = `${this.file}.records`; + const files = await readdir(directory).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }, + ); + await Promise.all( + files + .filter( + (file) => + /^[a-f0-9]{64}\.json(?:\.[a-f0-9-]+\.tmp)?$/.test( + file, + ) && + (file.endsWith(".tmp") || + !retained.has(file.slice(0, -5))), + ) + .map((file) => rm(path.join(directory, file), { force: true })), + ); + } +} diff --git a/ts/packages/copilot-macros/test/learningEvidence.spec.ts b/ts/packages/copilot-macros/test/learningEvidence.spec.ts index f035fa1361..dd8442f47e 100644 --- a/ts/packages/copilot-macros/test/learningEvidence.spec.ts +++ b/ts/packages/copilot-macros/test/learningEvidence.spec.ts @@ -134,6 +134,7 @@ it("provides body-free current recording facts only after physical verification" sizeMatchesRecordedMetadata: true, }, }); + expect(JSON.stringify(evidence)).not.toMatch( /https:|base64Content|YWJj|example.invalid/, ); @@ -151,6 +152,26 @@ it("provides body-free current recording facts only after physical verification" expect(() => verifiedLearningEvidence(trace, "trace")).toThrow(); }); +it("loads and verifies a learning trace without a preliminary evidence pass", async () => { + const { store, trace } = await persist(); + const verify = import.meta.jest.spyOn(store, "verify"); + const loaded = await store.verifyLearningTrace("trace"); + expect(loaded).toEqual(trace); + expect(verify).toHaveBeenCalledTimes(1); + expect(verifiedLearningEvidence(loaded, "trace")).toBeDefined(); +}); + +it("rejects unsupported trace capabilities on a learning-specific read", async () => { + const { store, trace } = await persist(); + await writeFile( + path.join(root, "traces", "trace.json"), + JSON.stringify({ ...trace, capabilities: ["unrecognized"] }), + ); + await expect(store.verifyLearningTrace("trace")).rejects.toThrow( + "Unsupported learning trace identity or capabilities.", + ); +}); + it.each([ "foreign physical trace", "missing original", diff --git a/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts b/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts index d995ffaaa6..fd4953dc56 100644 --- a/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts +++ b/ts/packages/copilot-macros/test/macroAuthoringRuntime.spec.ts @@ -237,7 +237,7 @@ describe("portable authoring boundary", () => { ], }; const source = - 'async function execute(api: RecipeApi, params: JsonValue): Promise { return await api.tools.call("read", params); }'; + 'async function execute(api: RecipeApi, params: JsonValue): Promise { if (Reflect.has(globalThis, "privateFixture")) throw Error("Leaked private fixture"); Reflect.set(globalThis, "privateFixture", params); return await api.tools.call("read", params); }'; const sourceQuery = jest.fn( async (_prompt: string, _signal: AbortSignal) => JSON.stringify({ source }), diff --git a/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts b/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts index 9b03d36d0f..5b08717574 100644 --- a/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts +++ b/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { @@ -14,6 +14,7 @@ import { MacroChildAuthoring, } from "../src/macroChildAuthoring.js"; import { sealMacroChildCapture } from "../src/macroChildCapture.js"; +import { RevisionRecords } from "../src/revisionRecords.js"; const jest = (import.meta as ImportMeta & { jest: typeof globalThis.jest }) .jest; @@ -48,7 +49,9 @@ describe("durable scoped Copilot authoring exchange", () => { }); const active = new Set(["job"]); beforeEach(async () => { - directory = await mkdtemp(path.join(os.tmpdir(), "child-authoring-")); + directory = await mkdtemp( + path.join(process.cwd(), ".child-authoring-"), + ); broker = new MacroChildAuthoring(directory); const token: RecordingToken = { id: "token", @@ -245,9 +248,11 @@ describe("durable scoped Copilot authoring exchange", () => { broker.query(job, "question", signal()), ).rejects.toBeInstanceOf(MacroAuthoringPending); const file = path.join(directory, "child-authoring.json"); - const saved = JSON.parse(await readFile(file, "utf8")); - saved[0].prompt = "modified"; - await writeFile(file, JSON.stringify(saved)); + const records = new RevisionRecords(file); + const saved = await records.load(); + const [id, request] = [...saved.entries()][0]; + (request as { prompt: string }).prompt = "modified"; + await records.commit(new Map([[id, request]])); await expect( new MacroChildAuthoring(directory).claim(claim(), active), ).rejects.toThrow("digest mismatch"); @@ -263,6 +268,131 @@ describe("durable scoped Copilot authoring exchange", () => { ).rejects.toThrow("missing or changed"); }); + it("arbitrates independent owners' competing leases and rejects the losing submission", async () => { + await expect( + broker.query(job, "competing question", signal()), + ).rejects.toBeInstanceOf(MacroAuthoringPending); + const other = new MacroChildAuthoring(directory); + const claims = await Promise.all([ + broker.claim(claim(), active), + other.claim({ ...claim(), workerId: "other" }, active), + ]); + expect(claims.filter(Boolean)).toHaveLength(1); + const winner = claims.find(Boolean)!; + await expect( + other.submit( + { + ...claim(), + workerId: claims[0] ? "other" : "worker", + ...winner, + outcome: { + status: "completed", + text: "answer", + childId: "child", + }, + }, + active, + ), + ).rejects.toThrow("scoped lease"); + }); + + it("does not wake idle IPC claimers on empty or already leased scans", async () => { + let notifications = 0; + const notifying = new MacroChildAuthoring( + directory, + "typeagent", + () => { + notifications++; + }, + ); + expect(await notifying.claim(claim(), active)).toBeUndefined(); + expect(notifications).toBe(0); + await expect( + notifying.query(job, "notification question", signal()), + ).rejects.toBeInstanceOf(MacroAuthoringPending); + expect(notifications).toBe(1); + expect(await notifying.claim(claim(), active)).toBeDefined(); + expect(notifications).toBe(2); + expect(await notifying.claim(claim(), active)).toBeUndefined(); + expect(notifications).toBe(2); + }); + + it("rejects missing immutable request payloads instead of treating a damaged canonical store as empty", async () => { + await expect( + broker.query(job, "retained request", signal()), + ).rejects.toBeInstanceOf(MacroAuthoringPending); + const file = path.join(directory, "child-authoring.json"); + const pointer = await readFile(file, "utf8"); + const digest = Object.values(JSON.parse(pointer).records)[0] as string; + await rm(path.join(`${file}.records`, `${digest}.json`)); + await expect( + new MacroChildAuthoring(directory).claim(claim(), active), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(file, "utf8")).toBe(pointer); + }); + + it("purges every obsolete request revision and orphan payload on forget", async () => { + const prompt = "private uniquely retained authoring prompt"; + await expect( + broker.query(job, prompt, signal()), + ).rejects.toBeInstanceOf(MacroAuthoringPending); + const request = (await broker.claim(claim(), active))!; + await broker.submit( + { + ...claim(), + ...request, + outcome: { + status: "completed", + text: "private answer", + childId: "child", + }, + }, + active, + ); + await broker.forget(job.jobId); + const files = await readdir( + path.join(directory, "child-authoring.json.records"), + ); + const bytes = await Promise.all( + files.map((file) => + readFile( + path.join(directory, "child-authoring.json.records", file), + "utf8", + ), + ), + ); + expect(bytes.join("")).not.toContain(prompt); + expect(bytes.join("")).not.toContain("private answer"); + expect( + await new MacroChildAuthoring(directory).claim(claim(), active), + ).toBeUndefined(); + }); + + it("finishes privacy cleanup after interruption between forget publication and payload purge", async () => { + await expect( + broker.query(job, "private interrupted purge", signal()), + ).rejects.toBeInstanceOf(MacroAuthoringPending); + const file = path.join(directory, "child-authoring.json"); + const records = new RevisionRecords(file); + const retained = await records.load(); + const digest = JSON.parse(await readFile(file, "utf8")).records[ + [...retained.keys()][0] + ] as string; + const staging = path.join( + `${file}.records`, + `${digest}.json.00000000-0000-0000-0000-000000000000.tmp`, + ); + await writeFile(staging, "private interrupted staged payload"); + await records.commit( + new Map([...retained.keys()].map((id) => [id, undefined])), + ); + expect((await readdir(`${file}.records`)).length).toBeGreaterThan(0); + expect( + await new MacroChildAuthoring(directory).claim(claim(), active), + ).toBeUndefined(); + expect(await readdir(`${file}.records`)).toEqual([]); + }); + it("rejects cancelled, secret-bearing and oversized requests before dispatch", async () => { await expect( broker.query(job, "x".repeat(1_048_577), signal()), diff --git a/ts/packages/copilot-macros/test/macroChildLearning.spec.ts b/ts/packages/copilot-macros/test/macroChildLearning.spec.ts index bb5dc0ca2e..b874043114 100644 --- a/ts/packages/copilot-macros/test/macroChildLearning.spec.ts +++ b/ts/packages/copilot-macros/test/macroChildLearning.spec.ts @@ -3,7 +3,6 @@ import { createHash } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; -import os from "node:os"; import path from "node:path"; import type { MacroLearningJob, @@ -82,7 +81,7 @@ describe.each(["typeagent", "copilot-local"])( } beforeEach(async () => { directory = await mkdtemp( - path.join(os.tmpdir(), "child-learning-"), + path.join(process.cwd(), ".child-learning-"), ); fallbackCalls = 0; manager = new MacroManager( @@ -176,6 +175,41 @@ describe.each(["typeagent", "copilot-local"])( cwd: directory, }); + it("wakes bounded IPC waits on durable changes and reconciles missed hints after reconnect", async () => { + const waitingForChange = manager.claimMacroAuthoringRequest({ + ...claim(), + observeOnly: true, + waitMs: 30_000, + }); + const summary = await recorded(); + await waitingForChange; + await waiting(summary.learningJobId!, "awaitingAuthor"); + // No subscriber is present for this persisted request. + await manager.close(); + manager = new MacroManager(directory, undefined, profileId); + await manager.configureLearning(runtime()); + await waiting(summary.learningJobId!, "awaitingAuthor"); + const request = await manager.claimMacroAuthoringRequest({ + ...claim(), + waitMs: 30_000, + }); + expect(request?.jobId).toBe(summary.learningJobId); + const cancellation = manager.claimMacroAuthoringRequest({ + ...claim(), + observeOnly: true, + waitMs: 30_000, + }); + await manager.cancelMacroLearningJob(summary.learningJobId!); + await cancellation; + expect( + (await manager.getMacroLearningJob(summary.learningJobId!)) + .status, + ).toBe("cancelled"); + expect( + await manager.claimMacroAuthoringRequest(claim()), + ).toBeUndefined(); + }); + it("reports a mismatched receipt as failed learning, not a completed recording or default-author fallback", async () => { await expect(recorded("another-recording")).rejects.toThrow( "does not correlate", diff --git a/ts/packages/copilot-macros/test/macroLearning.spec.ts b/ts/packages/copilot-macros/test/macroLearning.spec.ts index c50cf2d856..a7c78fc46b 100644 --- a/ts/packages/copilot-macros/test/macroLearning.spec.ts +++ b/ts/packages/copilot-macros/test/macroLearning.spec.ts @@ -12,10 +12,10 @@ import { type ReplayToolHost, } from "@typeagent/copilot-macros"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import path from "node:path"; import { macroProgramArtifact } from "./recipeCatalogFixtures.js"; +import { RevisionRecords } from "../src/revisionRecords.js"; const cwd = "C:\\learning-workspace"; const jest = (import.meta as ImportMeta & { jest: typeof globalThis.jest }) @@ -34,7 +34,9 @@ afterEach(async () => { }); async function setup(mode: MacroLearningMode = "prepare", readOnly?: boolean) { - const directory = await mkdtemp(path.join(os.tmpdir(), "macro-learning-")); + const directory = await mkdtemp( + path.join(process.cwd(), ".macro-learning-"), + ); directories.push(directory); const host: ReplayToolHost = { inspectTool: jest.fn(async (_server, toolName) => ({ @@ -51,6 +53,23 @@ async function setup(mode: MacroLearningMode = "prepare", readOnly?: boolean) { return { directory, manager, host }; } +async function mutateStoredJob( + directory: string, + jobId: string, + mutate: (job: { status: string; attempts: Record }) => void, +) { + const records = new RevisionRecords( + path.join(directory, "copilot-macros", "learning.json"), + ); + const state = await records.load(); + const job = state.get(jobId) as { + status: string; + attempts: Record; + }; + mutate(job); + await records.commit(new Map([[jobId, job]])); +} + function recordedTrace(sessionId: string): RecordedInteractionTrace { return { schemaVersion: 1, @@ -803,13 +822,10 @@ describe("durable macro learning", () => { it("recovers interrupted stages within a persisted attempt budget", async () => { const { manager, directory, host } = await setup("all"); const summary = await capture(manager, { learning: true }); - const file = path.join(directory, "copilot-macros", "learning.json"); - const state = JSON.parse(await readFile(file, "utf8")) as { - jobs: Array<{ status: string; attempts: { extract: number } }>; - }; - state.jobs[0].status = "extracting"; - state.jobs[0].attempts = { extract: 1 }; - await writeFile(file, JSON.stringify(state)); + await mutateStoredJob(directory, summary.learningJobId!, (job) => { + job.status = "extracting"; + job.attempts = { extract: 1 }; + }); const restarted = new MacroManager(directory, host); await restarted.configureLearning(runtime()); expect((await terminal(restarted, summary.learningJobId!)).status).toBe( @@ -959,19 +975,16 @@ describe("durable macro learning", () => { await manager.configureLearning(runtime()); const summary = await capture(manager, { learning: true }); const job = await terminal(manager, summary.learningJobId!); - const stateFile = path.join( - directory, - "copilot-macros", - "learning.json", - ); - const state = JSON.parse(await readFile(stateFile, "utf8")) as { - jobs: Array<{ status: string }>; - }; - state.jobs[0].status = "building"; - await writeFile(stateFile, JSON.stringify(state)); - await writeFile( - path.join(directory, "copilot-macros", "index.json"), - "[]", + await mutateStoredJob(directory, job.jobId, (stored) => { + stored.status = "building"; + }); + await rm( + path.join( + directory, + "copilot-macros", + "catalog", + `${job.macro!.macroId}.json`, + ), ); const restarted = new MacroManager(directory, host); const learner = runtime(); @@ -1010,17 +1023,10 @@ describe("durable macro learning", () => { it("never exceeds the persisted per-stage restart budget", async () => { const { manager, host, directory } = await setup("all"); const summary = await capture(manager, { learning: true }); - const stateFile = path.join( - directory, - "copilot-macros", - "learning.json", - ); - const state = JSON.parse(await readFile(stateFile, "utf8")) as { - jobs: Array<{ status: string; attempts: { extract: number } }>; - }; - state.jobs[0].status = "extracting"; - state.jobs[0].attempts = { extract: 2 }; - await writeFile(stateFile, JSON.stringify(state)); + await mutateStoredJob(directory, summary.learningJobId!, (job) => { + job.status = "extracting"; + job.attempts = { extract: 2 }; + }); const restarted = new MacroManager(directory, host); const learner = runtime(); await restarted.configureLearning(learner); diff --git a/ts/packages/copilot-macros/test/macroPrograms.spec.ts b/ts/packages/copilot-macros/test/macroPrograms.spec.ts index 68f0ff6ad2..6ff512fc3a 100644 --- a/ts/packages/copilot-macros/test/macroPrograms.spec.ts +++ b/ts/packages/copilot-macros/test/macroPrograms.spec.ts @@ -9,23 +9,30 @@ import { type RecordedInteractionTrace, } from "@typeagent/copilot-macros"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import os from "node:os"; +import { mkdir, mkdtemp, readdir, rename, rm, rmdir } from "node:fs/promises"; import path from "node:path"; +import { compileRecipeProgram } from "../src/recipeCompiler.js"; +import { RecipeExecutor } from "../src/recipeExecutor.js"; +import { recipeDigest, recipeSchemaDigest } from "../src/recipeCore.js"; +import type { RecipeHost } from "../src/recipeContracts.js"; import { recipeArtifact } from "./recipeCatalogFixtures.js"; +import { readProgramJournal } from "../src/macroProgramRun.js"; const cwd = "C:\\checkpoint-a"; let directory: string; +let executor: RecipeExecutor; beforeEach(async () => { - directory = await mkdtemp(path.join(os.tmpdir(), "macro-programs-")); + directory = await mkdtemp(path.join(process.cwd(), ".macro-programs-")); + executor = new RecipeExecutor({ maxWorkers: 2 }); }); afterEach(async () => { + await executor.close(); await rm(directory, { recursive: true, force: true }); }); -async function setup(mode: MacroLearningMode = "all") { +async function setup(mode: MacroLearningMode = "all", source?: string) { const manager = new MacroManager(directory, { inspectTool: async (_server, toolName) => ({ toolName, @@ -92,6 +99,33 @@ async function setup(mode: MacroLearningMode = "all") { validateGrammar: () => {}, buildProgram: async (macro) => recipeArtifact(macro.macroId, macro.sourceTraceId, { + ...(source + ? { + source: { + language: "typescript" as const, + text: source, + digest: recipeDigest(source), + }, + executable: compileRecipeProgram(source), + capabilities: [ + { + id: "read", + transport: "native" as const, + providerId: "test", + operation: "read", + contractVersion: "1", + inputSchema: { type: "null" as const }, + outputSchema: { type: "string" as const }, + schemaDigest: recipeSchemaDigest( + { type: "null" }, + { type: "string" }, + ), + compatibility: "exact" as const, + effect: "read" as const, + }, + ], + } + : {}), inputSchema: { type: "object", properties: { step_1_path: { type: "string" } }, @@ -135,6 +169,76 @@ async function setup(mode: MacroLearningMode = "all") { return { manager, ref: job.macro, job }; } +function realRuntime( + manager: MacroManager, + invoke: RecipeHost["invokeCapability"], +) { + manager.configureProgramRuntime({ + execute: (artifact, input, request) => + executor.execute( + artifact, + input, + { + runId: request.runId, + signal: request.signal, + binding: { + workspaceId: "workspace", + profileId: "profile", + providerId: "test", + accountId: "account", + sessionId: "session", + generation: "1", + }, + host: { + inspectCapability: async (capability) => ({ + status: "available", + capability, + }), + invokeCapability: invoke, + invokeSemantic: async () => ({ + status: "unsupported", + code: "test", + message: "No semantic calls", + }), + }, + }, + { + onCheckpoint: request.onCheckpoint, + timeoutMs: request.timeoutMs, + }, + ), + }); +} + +function deferred() { + let resolve!: () => void; + const promise = new Promise((finish) => { + resolve = finish; + }); + return { promise, resolve }; +} + +async function within(promise: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("Expected concurrent progress")), + 3_000, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +async function journal(runId: string) { + return readProgramJournal(path.join(directory, "copilot-macros"), runId); +} + function fakeRuntime(): MacroProgramRuntime { let calls = 0; return { @@ -146,18 +250,8 @@ function fakeRuntime(): MacroProgramRuntime { phase: "dispatched" as const, }; await context.onCheckpoint(checkpoint); - const journal = JSON.parse( - await readFile( - path.join( - directory, - "copilot-macros", - "program-runs", - `${context.runId}.json`, - ), - "utf8", - ), - ); - expect(journal.checkpoints[0].phase).toBe("dispatched"); + const saved = await journal(context.runId); + expect(saved.checkpoints[0].phase).toBe("dispatched"); calls++; const completed = { ...checkpoint, phase: "completed" as const }; await context.onCheckpoint(completed); @@ -178,6 +272,185 @@ function fakeRuntime(): MacroProgramRuntime { } describe("TypeAgent program macro integration", () => { + it("surfaces persistent journal publication failures without inventing durable completion", async () => { + const { manager, ref } = await setup( + "all", + `async function execute(api, params) { + const first = await api.tools.call("read", null); + return [first, await api.tools.call("read", null)].join(""); + }`, + ); + const runId = "failed-publication"; + const destination = path.join( + directory, + "copilot-macros", + "program-runs", + `${runId}.json`, + ); + const retained = `${destination}.retained`; + let calls = 0; + realRuntime(manager, async () => { + calls++; + await rename(destination, retained); + await mkdir(destination); + return { status: "completed", value: "fresh" }; + }); + try { + expect( + ( + await manager.runMacro({ + ...ref, + runId, + inputs: { step_1_path: "beta.txt" }, + }) + ).status, + ).toBe("failed"); + const record = await manager.getMacroRun(runId); + expect(record.program?.completedCallIds).toEqual([]); + expect(record.program?.checkpoints).toEqual([ + expect.objectContaining({ phase: "dispatched" }), + ]); + expect(calls).toBe(1); + expect( + (await readdir(path.dirname(destination))).filter((name) => + name.endsWith(".tmp"), + ), + ).toEqual([]); + } finally { + await rmdir(destination); + await rename(retained, destination); + await manager.close(); + } + }); + + it("overlaps real read calls only after durable dispatch and preserves reply identities across restart", async () => { + const { manager, ref } = await setup( + "all", + `async function execute(api, params) { + return (await Promise.all([api.tools.call("read", null), api.tools.call("read", null)])).join("|"); + }`, + ); + const entered = deferred(); + const secondFinished = deferred(); + const releases = [deferred(), deferred()]; + let calls = 0; + realRuntime(manager, async (_capability, _input, call) => { + const index = calls++; + expect((await journal("parallel")).checkpoints).toContainEqual( + expect.objectContaining({ + callId: call.callId, + phase: "dispatched", + }), + ); + if (calls === 2) entered.resolve(); + await releases[index].promise; + if (index === 1) secondFinished.resolve(); + return { status: "completed", value: `value-${index}` }; + }); + const run = manager.runMacro({ + ...ref, + runId: "parallel", + inputs: { step_1_path: "beta.txt" }, + }); + try { + await within( + Promise.race([ + entered.promise, + run.then((response) => { + throw new Error( + `Run settled before overlap: ${JSON.stringify(response)}`, + ); + }), + ]), + ); + releases[1].resolve(); + await within(secondFinished.promise); + releases[0].resolve(); + expect((await run).status).toBe("completed"); + const persisted = await new MacroManager(directory).getMacroRun( + "parallel", + ); + expect(persisted.result).toBe("value-0|value-1"); + expect(persisted.program?.completedCallIds).toEqual([ + "parallel:2", + "parallel:1", + ]); + expect((await journal("parallel")).checkpoints).toEqual([ + expect.objectContaining({ + callId: "parallel:1", + phase: "completed", + }), + expect.objectContaining({ + callId: "parallel:2", + phase: "completed", + }), + ]); + } finally { + releases.forEach((release) => release.resolve()); + await run; + await manager.close(); + } + }); + + it.each(["denied", "cancelled", "disabled"] as const)( + "stops unsent parallel reads after %s without accepting late outcomes", + async (terminal) => { + const { manager, ref } = await setup( + "all", + `async function execute(api, params) { + return (await Promise.all(Array.from({length: 8}, () => api.tools.call("read", null)))).join(""); + }`, + ); + const entered = deferred(); + const releases = Array.from({ length: 4 }, () => deferred()); + const signals: AbortSignal[] = []; + realRuntime(manager, async (_capability, _input, call) => { + const index = signals.length; + signals.push(call.signal); + if (signals.length === 4) entered.resolve(); + await releases[index].promise; + if (terminal === "denied" && index === 0) + return { + status: "denied", + code: "permission", + message: "Denied", + }; + return { status: "completed", value: "fresh" }; + }); + const runId = `parallel-${terminal}`; + const run = manager.runMacro({ + ...ref, + runId, + inputs: { step_1_path: "beta.txt" }, + }); + try { + await within(entered.promise); + if (terminal === "cancelled") manager.cancelMacroRun(runId); + else { + if (terminal === "disabled") + await manager.disableMacro(ref); + releases[0].resolve(); + } + const response = await within(run); + expect(response.status).not.toBe("completed"); + expect(signals).toHaveLength(4); + expect(signals.every((signal) => signal.aborted)).toBe(true); + const persisted = await manager.getMacroRun(runId); + expect(persisted.program?.completedCallIds).toEqual( + terminal === "disabled" ? [`${runId}:1`] : [], + ); + const before = await journal(runId); + releases.forEach((release) => release.resolve()); + await new Promise((resolve) => setImmediate(resolve)); + expect(await journal(runId)).toEqual(before); + expect(signals).toHaveLength(4); + } finally { + releases.forEach((release) => release.resolve()); + await run; + await manager.close(); + } + }, + ); it("publishes under All and runs fresh inputs without a runner or replay consent", async () => { const { manager, ref, job } = await setup(); expect(job.status).toBe("ready"); diff --git a/ts/packages/copilot-macros/test/macroStorage.spec.ts b/ts/packages/copilot-macros/test/macroStorage.spec.ts new file mode 100644 index 0000000000..36907ec137 --- /dev/null +++ b/ts/packages/copilot-macros/test/macroStorage.spec.ts @@ -0,0 +1,513 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { + appendFile, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { MacroManager } from "../src/macroManager.js"; +import { readProgramJournal, runMacroProgram } from "../src/macroProgramRun.js"; +import { + publishImmutable, + RevisionRecords, + withRecordLock, +} from "../src/revisionRecords.js"; +import type { + CopilotToolMacro, + MacroProgramRuntime, +} from "../src/contracts.js"; +import type { RecipeEffectCheckpoint } from "../src/recipeContracts.js"; +import { recipeArtifact } from "./recipeCatalogFixtures.js"; +import { induceMacroFromTrace } from "../src/macroDefinition.js"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +const jest = (import.meta as ImportMeta & { jest: typeof globalThis.jest }) + .jest; + +let directory: string; +beforeEach(async () => { + directory = await mkdtemp(path.join(process.cwd(), ".macro-storage-")); +}); +afterEach(async () => { + await rm(directory, { recursive: true, force: true }); +}); + +async function draft(id = "macro"): Promise { + const trace = { + schemaVersion: 1 as const, + sessionId: "session", + cwd: directory, + prompt: "read file", + response: "answer", + startedAt: "2026-10-19T00:00:00Z", + completedAt: "2026-10-19T00:00:01Z", + toolCalls: [ + { + toolCallId: "read", + name: "read", + mcpServerName: "files", + arguments: { path: "alpha.txt" }, + result: "contents", + status: "completed" as const, + }, + ], + }; + const macro = await induceMacroFromTrace( + "procedure:test", + trace, + id, + id, + "read file", + trace.completedAt, + { + inspectTool: async () => ({ + toolName: "read", + mcpServerName: "files", + schemaFingerprint: "v1", + }), + callTool: async () => { + throw new Error("Not dispatched"); + }, + }, + ); + macro.executionKind = "program"; + macro.program = recipeArtifact(id, "procedure:test", { + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + provenance: { + captureId: "procedure:test", + taskId: "procedure:test", + tokenId: "procedure:test", + evidenceIds: ["evidence"], + legacySource: { macroId: id, version: macro.version }, + }, + }); + return macro; +} + +describe("active macro record publication", () => { + it("does not expose staged revisions, survives interrupted publication, and retries the same immutable version", async () => { + const manager = new MacroManager(directory); + const macro = await draft(); + const pointer = path.join( + directory, + "copilot-macros", + "catalog", + "macro.json", + ); + const publish = jest + .spyOn( + manager as unknown as { + upsertSummary(macro: CopilotToolMacro): Promise; + }, + "upsertSummary", + ) + .mockRejectedValueOnce(new Error("interrupted publication")); + await expect(manager.saveDraft(macro)).rejects.toThrow(); + publish.mockRestore(); + expect(await manager.listMacros()).toEqual([]); + await manager.saveDraft(macro); + const version = path.join( + directory, + "copilot-macros", + "macros", + "macro", + "versions", + "1.json", + ); + const retained = await readFile(version, "utf8"); + const interrupted = { + ...macro, + version: 2, + description: "never published", + }; + await publishImmutable( + path.join(path.dirname(version), "2.json"), + JSON.stringify(interrupted), + ); + await writeFile( + `${pointer}.interrupted.tmp`, + JSON.stringify({ schemaVersion: 2, summary: interrupted }), + ); + const reopened = new MacroManager(directory); + expect( + (await reopened.inspectMacro({ macroId: "macro" })).version, + ).toBe(1); + expect(await readFile(version, "utf8")).toBe(retained); + await manager.deleteMacro({ macroId: "macro" }); + expect(await reopened.listMacros()).toEqual([]); + await expect(readFile(version)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect( + readFile(`${pointer}.interrupted.tmp`), + ).rejects.toMatchObject({ code: "ENOENT" }); + await manager.close(); + await reopened.close(); + }); + + it("updates one production catalog pointer without rewriting other macros or version history", async () => { + const manager = new MacroManager(directory); + const macros = await Promise.all( + Array.from({ length: 100 }, (_, index) => draft(`macro-${index}`)), + ); + for (const macro of macros) await manager.saveDraft(macro); + const root = path.join(directory, "copilot-macros", "catalog"); + const files = await readdir(root); + const before = new Map( + await Promise.all( + files.map( + async (file) => + [ + file, + await readFile(path.join(root, file), "utf8"), + ] as const, + ), + ), + ); + await manager.saveDraft({ ...macros[0], version: 2 }); + const changed = []; + for (const file of files) + if ( + before.get(file) !== + (await readFile(path.join(root, file), "utf8")) + ) + changed.push(file); + expect(changed).toEqual(["macro-0.json"]); + const candidateBytes = (await stat(path.join(root, "macro-0.json"))) + .size; + const baselineBytes = Buffer.byteLength( + JSON.stringify( + await manager.listMacros({ limit: 500 }), + undefined, + 2, + ), + ); + expect(candidateBytes).toBeLessThan(baselineBytes / 50); + process.stdout.write( + JSON.stringify({ + workload: "one-catalog-entry-among-100", + baselineJsonBytes: baselineBytes, + candidateJsonBytes: candidateBytes, + baselinePointerWrites: 1, + candidatePointerWrites: 1, + unrelatedPointersRewritten: 0, + }) + "\n", + ); + await manager.close(); + }); + + it("rejects incompatible canonical catalogs, learning state and child records without deleting bytes", async () => { + const root = path.join(directory, "copilot-macros"); + await mkdir(root); + const legacy = path.join(root, "index.json"); + await writeFile(legacy, "[]"); + await expect(new MacroManager(directory).listMacros()).rejects.toThrow( + "reset/reimport", + ); + expect(await readFile(legacy, "utf8")).toBe("[]"); + for (const file of ["learning.json", "child-authoring.json"]) { + const destination = path.join(root, file); + const bytes = JSON.stringify({ schemaVersion: 1, jobs: [] }); + await writeFile(destination, bytes); + await expect( + new RevisionRecords(destination).load(), + ).rejects.toThrow("reset/reimport"); + expect(await readFile(destination, "utf8")).toBe(bytes); + } + }); + + it("publishes competing writers once, rejects stale revisions, and leaves failed payloads unselected", async () => { + const file = path.join(directory, "records.json"); + const first = new RevisionRecords(file); + const second = new RevisionRecords(file); + await Promise.all([first.load(), second.load()]); + const results = await Promise.allSettled([ + first.commit(new Map([["first", { value: "first" }]])), + second.commit(new Map([["second", { value: "second" }]])), + ]); + expect( + results.filter((result) => result.status === "fulfilled"), + ).toHaveLength(1); + const current = new RevisionRecords(file); + const records = await current.load(); + expect(records.size).toBe(1); + const orphan = JSON.stringify({ value: "interrupted payload" }); + const digest = createHash("sha256").update(orphan).digest("hex"); + await publishImmutable( + path.join(`${file}.records`, `${digest}.json`), + orphan, + ); + expect([ + ...(await new RevisionRecords(file).load()).values(), + ]).not.toContainEqual(JSON.parse(orphan)); + await current.purgeUnreferenced(); + await expect( + readFile(path.join(`${file}.records`, `${digest}.json`)), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("releases the owner transaction on failed mutation", async () => { + const file = path.join(directory, "owner"); + await expect( + withRecordLock(file, async () => { + throw new Error("interrupted"); + }), + ).rejects.toThrow("interrupted"); + expect(await withRecordLock(file, async () => "recovered")).toBe( + "recovered", + ); + }); + + it("recovers the owner lock after process death inside a publication transaction", async () => { + const file = path.join(directory, "terminated-owner"); + const module = new URL("../revisionRecords.js", import.meta.url).href; + const { stdout } = await promisify(execFile)(process.execPath, [ + "--no-warnings", + "--input-type=module", + "-e", + `import { withRecordLock } from ${JSON.stringify(module)}; await withRecordLock(${JSON.stringify(file)}, async () => { process.stdout.write("locked"); process.exit(0); });`, + ]); + expect(stdout).toBe("locked"); + expect(await withRecordLock(file, async () => "recovered")).toBe( + "recovered", + ); + }); + + it("mutates one active production job without rewriting unrelated retained job payloads", async () => { + const root = path.join(directory, "copilot-macros"); + const file = path.join(root, "learning.json"); + const store = new RevisionRecords(file); + const cwd = path.resolve(directory).toLowerCase(); + const now = "2026-10-19T00:00:00Z"; + const jobs = Array.from({ length: 100 }, (_, index) => ({ + jobId: createHash("sha256").update(String(index)).digest("hex"), + fingerprint: createHash("sha256") + .update(`fingerprint-${index}`) + .digest("hex"), + traceId: `trace-${index}`, + cwd, + sessionId: "source", + mode: "all", + status: "failed", + preferenceRevision: 1, + attempts: {}, + verifierAnswers: { + ["0".repeat(64)]: "retained verifier answer ".repeat(350), + }, + createdAt: now, + updatedAt: now, + })); + const metadata = { + schemaVersion: 1, + preferences: [{ cwd, mode: "all", revision: 1 }], + suppressed: [], + suppressedMacroIds: [], + }; + await store.commit( + new Map([ + ["$state", metadata], + ...jobs.map((job) => [job.jobId, job] as [string, unknown]), + ]), + ); + const manager = new MacroManager(directory); + await manager.getMacroLearningJob(jobs[0].jobId); + const before = JSON.parse(await readFile(file, "utf8")) as { + records: Record; + }; + const filesBefore = new Set(await readdir(`${file}.records`)); + await manager.cancelMacroLearningJob(jobs[0].jobId); + const after = JSON.parse(await readFile(file, "utf8")) as { + records: Record; + }; + expect( + Object.keys(after.records).filter( + (key) => before.records[key] !== after.records[key], + ), + ).toEqual([jobs[0].jobId]); + const added = (await readdir(`${file}.records`)).filter( + (entry) => !filesBefore.has(entry), + ); + expect(added).toHaveLength(1); + const afterBytes = + (await stat(file)).size + + (await stat(path.join(`${file}.records`, added[0]))).size; + const changed = JSON.parse( + await readFile(path.join(`${file}.records`, added[0]), "utf8"), + ); + const baselineBytes = Buffer.byteLength( + JSON.stringify({ ...metadata, jobs: [changed, ...jobs.slice(1)] }), + ); + expect(afterBytes).toBeLessThan(baselineBytes / 10); + process.stdout.write( + JSON.stringify({ + workload: "one-job-among-100", + baselineJsonBytes: baselineBytes, + candidateJsonBytes: afterBytes, + baselineJsonWrites: 1, + candidateJsonWrites: 2, + unrelatedPayloadsRewritten: 0, + }) + "\n", + ); + await manager.close(); + }); +}); + +describe("append-only production program journal", () => { + it("measures cumulative bytes against the previous full-snapshot algorithm while preserving durable dispatch and run identity", async () => { + const macro = await draft(); + const runId = "measured"; + const baseline = { + macroId: macro.macroId, + version: macro.version, + runId, + startedAt: new Date().toISOString(), + checkpoints: [] as RecipeEffectCheckpoint[], + }; + let baselineBytes = Buffer.byteLength(JSON.stringify(baseline)); + let checkpoints = 0; + let calls = 0; + const runtime: MacroProgramRuntime = { + async execute(artifact, _input, context) { + for (let index = 0; index < 100; index++) { + for (const phase of ["dispatched", "completed"] as const) { + const checkpoint: RecipeEffectCheckpoint = { + callId: `call-${index}`, + capabilityId: "read", + effect: "read", + phase, + }; + await context.onCheckpoint(checkpoint); + const durable = await readProgramJournal( + directory, + runId, + ); + expect(durable.runId).toBe(runId); + expect(durable.checkpoints[index]).toEqual(checkpoint); + if (phase === "dispatched") calls++; + if (phase === "dispatched") + baseline.checkpoints.push(checkpoint); + else baseline.checkpoints[index] = checkpoint; + baselineBytes += Buffer.byteLength( + JSON.stringify(baseline), + ); + checkpoints++; + } + } + return { + runId, + recipeId: artifact.recipeId, + revision: artifact.revision, + accepted: true, + status: "completed", + checkpoints: baseline.checkpoints, + completedCallIds: baseline.checkpoints.map( + (checkpoint) => checkpoint.callId, + ), + semanticEvidence: [], + fallback: "never", + }; + }, + }; + const run = await runMacroProgram( + directory, + macro, + { + macroId: macro.macroId, + version: macro.version, + runId, + inputs: {}, + }, + runtime, + new AbortController().signal, + async () => {}, + ); + expect(run.status).toBe("completed"); + expect(calls).toBe(100); + const candidateBytes = ( + await stat(path.join(directory, "program-runs", `${runId}.json`)) + ).size; + expect(candidateBytes).toBeLessThan(baselineBytes / 20); + process.stdout.write( + JSON.stringify({ + workload: "100-two-phase-calls", + baselineJsonBytes: baselineBytes, + candidateJsonBytes: candidateBytes, + baselineWrites: checkpoints + 1, + candidateWrites: checkpoints + 1, + baselineRenames: checkpoints, + candidateRenames: 0, + durableDispatches: calls, + }) + "\n", + ); + await expect( + runMacroProgram( + directory, + macro, + { + macroId: macro.macroId, + version: macro.version, + runId, + inputs: {}, + }, + runtime, + new AbortController().signal, + async () => {}, + ), + ).rejects.toMatchObject({ code: "EEXIST" }); + }); + + it("ignores only an interrupted trailing checkpoint and recovers dispatched effects as uncertain", async () => { + const root = path.join(directory, "copilot-macros"); + const journalDirectory = path.join(root, "program-runs"); + await mkdir(journalDirectory, { recursive: true }); + const file = path.join(journalDirectory, "interrupted.json"); + const header = { + schemaVersion: 2, + macroId: "macro", + version: 1, + runId: "interrupted", + startedAt: new Date().toISOString(), + checkpoints: [], + }; + const checkpoint = { + callId: "call", + capabilityId: "read", + effect: "read", + phase: "dispatched", + }; + await writeFile( + file, + `${JSON.stringify(header)}\n${JSON.stringify({ checkpoint })}\n{"checkpoint":{"callId":"call","phase":"completed"`, + ); + const manager = new MacroManager(directory); + expect(await manager.getMacroRun("interrupted")).toMatchObject({ + runId: "interrupted", + error: { code: "execution_uncertain" }, + result: { checkpoints: [checkpoint] }, + }); + const retained = `${file}.retained`; + await rename(file, retained); + await writeFile(file, JSON.stringify({ ...header, schemaVersion: 1 })); + await expect(readProgramJournal(root, "interrupted")).rejects.toThrow( + "reset/reimport", + ); + await rm(file); + await rename(retained, file); + await appendFile(file, "\n"); + await expect(readProgramJournal(root, "interrupted")).rejects.toThrow(); + await manager.close(); + }); +}); diff --git a/ts/packages/copilot-macros/test/recipeExecutor.spec.ts b/ts/packages/copilot-macros/test/recipeExecutor.spec.ts index 6f0afe41d8..7badd691b5 100644 --- a/ts/packages/copilot-macros/test/recipeExecutor.spec.ts +++ b/ts/packages/copilot-macros/test/recipeExecutor.spec.ts @@ -85,7 +85,230 @@ const readSource = `async function execute(api, params) { return await api.tools.call("read", null); }`; +function deferred() { + let resolve!: () => void; + const promise = new Promise((finish) => { + resolve = finish; + }); + return { promise, resolve }; +} + +async function within(promise: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("Expected concurrent progress")), + 3_000, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +} + describe("contained recipe executor", () => { + it("overlaps independent reads and correlates reversed replies", async () => { + const entered = deferred(); + const secondFinished = deferred(); + const releases = [deferred(), deferred()]; + let calls = 0; + const context = execution(async () => { + const index = calls++; + if (calls === 2) entered.resolve(); + await releases[index].promise; + if (index === 1) secondFinished.resolve(); + return { status: "completed", value: `value-${index}` }; + }); + const run = executeRecipeProgram( + program(`async function execute(api, params) { + return (await Promise.all([api.tools.call("read", null), api.tools.call("read", null)])).join("|"); + }`), + null, + context, + ); + try { + await within(entered.promise); + releases[1].resolve(); + await within(secondFinished.promise); + releases[0].resolve(); + const result = await run; + expect(result.status).toBe("completed"); + expect(result.output).toBe("value-0|value-1"); + expect(result.completedCallIds).toEqual(["run:2", "run:1"]); + } finally { + releases.forEach((release) => release.resolve()); + await run; + } + }); + + it("bounds active read calls and stops queued work after cancellation", async () => { + const entered = deferred(); + const release = deferred(); + const controller = new AbortController(); + const signals: AbortSignal[] = []; + const context = execution(async (_capability, _input, call) => { + signals.push(call.signal); + if (signals.length === 4) entered.resolve(); + await release.promise; + return { status: "completed", value: "late" }; + }); + context.signal = controller.signal; + const run = executeRecipeProgram( + program(`async function execute(api, params) { + return (await Promise.all(Array.from({length: 8}, () => api.tools.call("read", null)))).join(""); + }`), + null, + context, + ); + try { + await within(entered.promise); + expect(signals).toHaveLength(4); + controller.abort(); + const result = await within(run); + expect(result.status).toBe("cancelled"); + expect(result.completedCallIds).toEqual([]); + expect(signals.every((signal) => signal.aborted)).toBe(true); + release.resolve(); + await new Promise((resolve) => setImmediate(resolve)); + expect(signals).toHaveLength(4); + expect(result.completedCallIds).toEqual([]); + } finally { + controller.abort(); + release.resolve(); + await run; + } + }); + + it("keeps effectful calls serialized even when guest code starts them together", async () => { + const firstEntered = deferred(); + const release = deferred(); + let calls = 0; + const context = execution(async () => { + if (++calls === 1) { + firstEntered.resolve(); + await release.promise; + } + return { status: "completed", value: "x" }; + }); + const run = executeRecipeProgram( + program( + `async function execute(api, params) { + return (await Promise.all([api.tools.call("read", null), api.tools.call("read", null)])).join(""); + }`, + { capabilities: [{ ...capability, effect: "write" }] }, + ), + null, + context, + ); + try { + await within(firstEntered.promise); + await new Promise((resolve) => setImmediate(resolve)); + expect(calls).toBe(1); + } finally { + release.resolve(); + } + expect((await run).output).toBe("xx"); + expect(calls).toBe(2); + }); + + it("runs semantic calls as barriers between read groups", async () => { + const firstEntered = deferred(); + const firstRelease = deferred(); + const semanticEntered = deferred(); + const semanticRelease = deferred(); + let reads = 0; + const context = execution(async () => { + if (++reads === 1) { + firstEntered.resolve(); + await firstRelease.promise; + } + return { status: "completed", value: "read" }; + }); + context.host.invokeSemantic = async () => { + semanticEntered.resolve(); + await semanticRelease.promise; + return { + status: "completed", + value: "semantic", + evidence: { + functionId: "select", + functionRevision: "1", + resolvedModel: "actual", + latencyMs: 1, + validation: "passed", + sourceIds: [], + }, + }; + }; + const artifact = program( + `async function execute(api, params) { + return (await Promise.all([api.tools.call("read", null), api.semantic.call("select", null), api.tools.call("read", null)])).join("|"); + }`, + { + execution: "hybrid", + semanticFunctions: semanticProgram().semanticFunctions.map( + (definition) => ({ + ...definition, + bounds: { ...definition.bounds, timeoutMs: 2_000 }, + }), + ), + }, + ); + const run = executeRecipeProgram(artifact, null, context); + try { + await within(firstEntered.promise); + expect(reads).toBe(1); + firstRelease.resolve(); + await within(semanticEntered.promise); + expect(reads).toBe(1); + semanticRelease.resolve(); + expect((await run).output).toBe("read|semantic|read"); + } finally { + firstRelease.resolve(); + semanticRelease.resolve(); + await run; + } + }); + + it("preflights in bounded parallel batches and selects failures in declaration order", async () => { + const capabilities = Array.from({ length: 6 }, (_, index) => ({ + ...capability, + id: `read-${index}`, + operation: `read-${index}`, + })); + const artifact = program(readSource, { capabilities }); + let active = 0; + let maxActive = 0; + const entered: string[] = []; + const releases: Array<() => void> = []; + const context = execution(); + context.host.inspectCapability = async (requirement) => { + entered.push(requirement.id); + maxActive = Math.max(maxActive, ++active); + await new Promise((resolve) => releases.push(resolve)); + active--; + return { status: "unavailable", message: requirement.id }; + }; + const run = executeRecipeProgram(artifact, null, context); + try { + for (let i = 0; i < 100 && entered.length < 4; i++) + await new Promise((resolve) => setImmediate(resolve)); + expect(entered).toEqual(["read-0", "read-1", "read-2", "read-3"]); + expect(maxActive).toBe(4); + } finally { + for (const release of releases.reverse()) release(); + } + const result = await run; + expect(result.status).toBe("unavailable"); + expect(result.message).toBe("read-0"); + expect(entered).toHaveLength(4); + expect(result.completedCallIds).toEqual([]); + }); + it("executes fresh host calls on both runs and awaits durable checkpoints", async () => { let calls = 0; const events: string[] = []; diff --git a/ts/packages/copilot-macros/test/recipeWorkerPool.spec.ts b/ts/packages/copilot-macros/test/recipeWorkerPool.spec.ts new file mode 100644 index 0000000000..4c0f2043df --- /dev/null +++ b/ts/packages/copilot-macros/test/recipeWorkerPool.spec.ts @@ -0,0 +1,404 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { compileRecipeProgram } from "../src/recipeCompiler.js"; +import { + configureRecipeExecutorAssets, + RecipeExecutor, +} from "../src/recipeExecutor.js"; +import { recipeDigest, recipeSchemaDigest } from "../src/recipeCore.js"; +import type { RecipeExecutionContext } from "../src/recipeRouting.js"; +import { recipeArtifact } from "./recipeCatalogFixtures.js"; + +const workerUrl = new URL("../recipeExecutorWorker.js", import.meta.url); +const source = `async function execute(api, params) { + if (Reflect.has(globalThis, "privateValue")) throw Error("Leaked guest state"); + Reflect.set(globalThis, "privateValue", params); + return params; +}`; + +function program(text = source) { + return recipeArtifact("warm", "capture", { + source: { language: "typescript", text, digest: recipeDigest(text) }, + executable: compileRecipeProgram(text), + inputSchema: { type: "string" }, + capabilities: [], + }); +} + +function context(): RecipeExecutionContext { + return { + runId: "warm", + signal: new AbortController().signal, + binding: { + workspaceId: "workspace", + profileId: "profile", + providerId: "test", + accountId: "account", + sessionId: "session", + generation: "1", + }, + host: { + inspectCapability: async (capability) => ({ + status: "available", + capability, + }), + invokeCapability: async () => ({ + status: "completed", + value: "fresh", + }), + invokeSemantic: async () => ({ + status: "unsupported", + code: "fixture", + message: "No semantic calls", + }), + }, + }; +} + +function deferred() { + let resolve!: () => void; + const promise = new Promise((finish) => { + resolve = finish; + }); + return { promise, resolve }; +} + +async function within(promise: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("Expected concurrent progress")), + 3_000, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +describe("owned warm recipe workers", () => { + let executor: RecipeExecutor; + let directory: string; + + beforeEach(async () => { + executor = new RecipeExecutor({ maxWorkers: 1 }); + directory = await mkdtemp(path.join(os.tmpdir(), "recipe-worker-")); + }); + + afterEach(async () => { + await executor.close(); + configureRecipeExecutorAssets({ workerUrl }); + await rm(directory, { recursive: true, force: true }); + }); + + it("initializes WASM once and isolates every private fixture context", async () => { + const counter = path.join(directory, "initializations.txt"); + const loader = path.join(directory, "quickjs.cjs"); + const modulePath = createRequire(import.meta.url).resolve( + "@tootallnate/quickjs-emscripten", + ); + await writeFile( + loader, + ` + const original = require(${JSON.stringify(modulePath)}); + module.exports.newQuickJSWASMModule = async () => { + require("node:fs").appendFileSync(${JSON.stringify(counter)}, "init\\n"); + return original.newQuickJSWASMModule(); + }; + `, + ); + configureRecipeExecutorAssets({ workerUrl, quickjsModulePath: loader }); + const artifact = program(); + for (let index = 0; index < 6; index++) { + const result = await executor.execute( + artifact, + `private-${index}`, + context(), + ); + expect(result.status).toBe("completed"); + expect(result.output).toBe(`private-${index}`); + expect(result.performance?.workerReused).toBe(index > 0); + } + expect(await readFile(counter, "utf8")).toBe("init\n"); + }); + + it.each([ + `async function execute(api, params): Promise { throw Error("failed fixture"); }`, + `async function execute(api, params): Promise { while (true) {} }`, + `async function execute(api, params): Promise { const values = []; while (true) values.push("x".repeat(100000)); }`, + `async function execute(api, params) { return new Promise(() => {}); }`, + ])( + "replaces failed workers before subsequent execution: %s", + async (text) => { + expect( + (await executor.execute(program(), "first", context())).status, + ).toBe("completed"); + const failed = await executor.execute( + program(text), + "failed", + context(), + { + timeoutMs: 500, + memoryLimitBytes: 2 * 1024 * 1024, + }, + ); + expect(failed.status).not.toBe("completed"); + const recovered = await executor.execute( + program(), + "fresh", + context(), + ); + expect(recovered.status).toBe("completed"); + expect(recovered.output).toBe("fresh"); + expect(recovered.performance?.workerReused).toBe(false); + }, + ); + + it("retires cached workers when assets change and recovers after a worker crash", async () => { + const artifact = program(); + await executor.execute(artifact, "before", context()); + configureRecipeExecutorAssets({ + workerUrl: new URL("data:text/javascript,process.exit(2)"), + }); + expect( + (await executor.execute(artifact, "crash", context())).status, + ).toBe("failed"); + configureRecipeExecutorAssets({ workerUrl }); + const recovered = await executor.execute(artifact, "after", context()); + expect(recovered.status).toBe("completed"); + expect(recovered.performance?.workerReused).toBe(false); + }); + + it("recovers from module initialization failures after asset repair", async () => { + configureRecipeExecutorAssets({ + workerUrl, + quickjsModulePath: path.join(directory, "missing-quickjs.cjs"), + }); + expect( + (await executor.execute(program(), "missing", context())).status, + ).toBe("failed"); + configureRecipeExecutorAssets({ workerUrl }); + const recovered = await executor.execute( + program(), + "repaired", + context(), + ); + expect(recovered.output).toBe("repaired"); + expect(recovered.performance?.workerReused).toBe(false); + }); + + it.each([ + { kind: "result", text: "not-json" }, + { + kind: "request", + text: JSON.stringify({ + request: 2, + kind: "tools", + id: "read", + input: null, + }), + }, + ])( + "rejects malformed worker replies and replaces the worker: $kind", + async (message) => { + const script = `import { parentPort } from "node:worker_threads"; + parentPort.once("message", () => parentPort.postMessage(${JSON.stringify(message)}));`; + configureRecipeExecutorAssets({ + workerUrl: new URL( + `data:text/javascript,${encodeURIComponent(script)}`, + ), + }); + expect( + (await executor.execute(program(), "invalid", context())) + .status, + ).toBe("failed"); + configureRecipeExecutorAssets({ workerUrl }); + expect( + (await executor.execute(program(), "recovered", context())) + .output, + ).toBe("recovered"); + }, + ); + + function blockedRun() { + const entered = deferred(); + const release = deferred(); + const execution = context(); + const capability = { + id: "read", + transport: "native" as const, + providerId: "test", + operation: "read", + contractVersion: "1", + inputSchema: { type: "null" as const }, + outputSchema: { type: "string" as const }, + schemaDigest: recipeSchemaDigest( + { type: "null" }, + { type: "string" }, + ), + compatibility: "exact" as const, + effect: "read" as const, + }; + const text = `async function execute(api, params) { return api.tools.call("read", null); }`; + const base = program(text); + const artifact = recipeArtifact("blocked", "capture", { + source: base.source, + executable: base.executable, + inputSchema: { type: "string" }, + capabilities: [capability], + }); + execution.host.invokeCapability = async () => { + entered.resolve(); + await release.promise; + return { status: "completed", value: "released" }; + }; + return { artifact, execution, entered, release }; + } + + it("bounds admission, cancels queued waits independently and records queue time", async () => { + const block = blockedRun(); + const first = executor.execute( + block.artifact, + "first", + block.execution, + ); + await within(block.entered.promise); + const controller = new AbortController(); + const cancelled = executor.execute(program(), "cancelled", { + ...context(), + signal: controller.signal, + }); + const third = executor.execute(program(), "third", context()); + const expired = executor.execute(program(), "expired", context(), { + timeoutMs: 100, + }); + controller.abort(); + expect((await within(cancelled)).status).toBe("cancelled"); + expect((await within(expired)).status).toBe("limit_exceeded"); + block.release.resolve(); + expect((await first).status).toBe("completed"); + const result = await third; + expect(result.output).toBe("third"); + expect(result.performance?.workerReused).toBe(true); + expect(result.performance!.workerQueueMs).toBeGreaterThan(0); + }); + + it("retains the measured eight-run concurrent capacity", async () => { + await executor.close(); + executor = new RecipeExecutor(); + const block = blockedRun(); + const entered = deferred(); + let active = 0; + block.execution.host.invokeCapability = async () => { + if (++active === 8) entered.resolve(); + await block.release.promise; + return { status: "completed", value: "fresh" }; + }; + const runs = Array.from({ length: 8 }, () => + executor.execute(block.artifact, "parallel", block.execution), + ); + try { + await within(entered.promise); + expect(active).toBe(8); + } finally { + block.release.resolve(); + } + expect( + (await Promise.all(runs)).every( + (result) => result.status === "completed", + ), + ).toBe(true); + }); + + it("lets active work finish on its captured assets while queued work uses new assets", async () => { + const block = blockedRun(); + const first = executor.execute( + block.artifact, + "first", + block.execution, + ); + await within(block.entered.promise); + const next = executor.execute(program(), "next", context()); + configureRecipeExecutorAssets({ workerUrl }); + block.release.resolve(); + expect((await first).status).toBe("completed"); + const result = await next; + expect(result.output).toBe("next"); + expect(result.performance?.workerReused).toBe(false); + }); + + it("rejects excess queued work explicitly instead of growing admission without bounds", async () => { + const block = blockedRun(); + const first = executor.execute( + block.artifact, + "first", + block.execution, + ); + await within(block.entered.promise); + const artifact = program(); + const queued = Array.from({ length: 65 }, () => + executor.execute(artifact, "queued", context()), + ); + const overflow = await within(queued[64]); + expect(overflow.status).toBe("failed"); + expect(overflow.message).toContain("admission queue is full"); + await executor.close(); + expect((await first).status).toBe("cancelled"); + expect(await Promise.all(queued.slice(0, 64))).toEqual( + Array.from({ length: 64 }, () => + expect.objectContaining({ status: "cancelled" }), + ), + ); + block.release.resolve(); + }); + + it("close cancels active and queued work and is idempotent", async () => { + const block = blockedRun(); + const first = executor.execute( + block.artifact, + "first", + block.execution, + ); + await within(block.entered.promise); + const queued = executor.execute(program(), "queued", context()); + await within(executor.close()); + expect((await first).status).toBe("cancelled"); + expect((await queued).status).toBe("cancelled"); + await executor.close(); + await expect( + executor.execute(program(), "late", context()), + ).rejects.toThrow("closed"); + expect(block.execution.signal.aborted).toBe(false); + block.release.resolve(); + }); + + it("keeps repeated fresh-context RSS growth within a 64 MiB soak budget", async () => { + const artifact = program(); + const observations: number[] = []; + for (let index = 0; index < 120; index++) { + const result = await executor.execute( + artifact, + `private-${index}`, + context(), + ); + expect(result.output).toBe(`private-${index}`); + if (index >= 20) { + expect(result.performance?.workerReused).toBe(true); + observations.push(process.memoryUsage().rss); + } + } + expect( + Math.max(...observations.slice(-20)) - + Math.min(...observations.slice(0, 20)), + ).toBeLessThanOrEqual(64 * 1024 * 1024); + }); +}); diff --git a/ts/packages/copilot-plugin/src/extension/macro-authoring.ts b/ts/packages/copilot-plugin/src/extension/macro-authoring.ts index 547024e451..b549f32814 100644 --- a/ts/packages/copilot-plugin/src/extension/macro-authoring.ts +++ b/ts/packages/copilot-plugin/src/extension/macro-authoring.ts @@ -18,14 +18,14 @@ type AuthoringConnection = Pick< | "close" >; -function waitForAuthoringPoll(signal: AbortSignal): Promise { +function waitForAuthoringReconnect(signal: AbortSignal): Promise { return new Promise((resolve) => { const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; - const timer = setTimeout(done, 1000); + const timer = setTimeout(done, 5_000); timer.unref(); signal.addEventListener("abort", done, { once: true }); if (signal.aborted) done(); @@ -35,8 +35,9 @@ function waitForAuthoringPoll(signal: AbortSignal): Promise { async function callAuthoringBackend( operation: () => Promise, signal: AbortSignal, + timeoutMs = 10_000, ): Promise { - const deadline = AbortSignal.any([signal, AbortSignal.timeout(10_000)]); + const deadline = AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]); deadline.throwIfAborted(); let interrupt!: () => void; const interrupted = new Promise((_resolve, reject) => { @@ -111,6 +112,21 @@ export class MacroAuthoringWorker { } private async drain(signal: AbortSignal): Promise { + while (!signal.aborted && Date.now() < this.until) { + try { + await this.drainConnection(signal); + return; + } catch (error) { + if (signal.aborted) return; + this.options.report( + `Copilot authoring connection interrupted; reconnecting for durable reconciliation: ${String(redactTraceValue(error instanceof Error ? error.message : String(error)))}`, + ); + await waitForAuthoringReconnect(signal); + } + } + } + + private async drainConnection(signal: AbortSignal): Promise { const connection = await this.options.connect(); try { if ( @@ -130,13 +146,14 @@ export class MacroAuthoringWorker { cwd: this.options.cwd, sessionId: this.options.sessionId, workerId: this.workerId, + waitMs: 30_000, }; const request = await callAuthoringBackend( () => connection.claimMacroAuthoringRequest(claim), signal, + 35_000, ); if (!request) { - await waitForAuthoringPoll(signal); continue; } const outcome = await this.author(request, connection, signal); @@ -222,6 +239,22 @@ export class MacroAuthoringWorker { ): Promise { while (!signal.aborted) { try { + // Register the IPC wait before reading authority so a concurrent + // cancellation cannot fall between reconciliation and waiting. + const changed = callAuthoringBackend( + () => + connection.claimMacroAuthoringRequest({ + cwd: this.options.cwd, + sessionId: this.options.sessionId, + workerId: this.workerId, + observeOnly: true, + waitMs: 30_000, + }), + signal, + 35_000, + ); + // Cancellation can end monitoring before this wait resolves. + void changed.catch(() => {}); const [preference, job] = await callAuthoringBackend( () => Promise.all([ @@ -244,11 +277,11 @@ export class MacroAuthoringWorker { return new Error( "Learning was disabled or the authoring job is no longer active.", ); + await changed; } catch (error) { if (signal.aborted) return undefined; throw error; } - await waitForAuthoringPoll(signal); } return undefined; } diff --git a/ts/packages/copilot-plugin/src/extension/native-recipe-service.ts b/ts/packages/copilot-plugin/src/extension/native-recipe-service.ts index b26222282a..0a4ca8b8b2 100644 --- a/ts/packages/copilot-plugin/src/extension/native-recipe-service.ts +++ b/ts/packages/copilot-plugin/src/extension/native-recipe-service.ts @@ -4,7 +4,7 @@ import type { CopilotSession } from "@github/copilot-sdk"; import { createMacroLearningRuntime } from "@typeagent/copilot-macros/macroAuthoringRuntime"; import { createMacroProgramAuthor } from "@typeagent/copilot-macros/macroProgramAuthoring"; -import { executeRecipeProgram } from "@typeagent/copilot-macros/recipeExecutor"; +import { RecipeExecutor } from "@typeagent/copilot-macros/recipeExecutor"; import { RecipeSemanticAdapter } from "@typeagent/copilot-macros/recipeSemantic"; import { getRecipeInferenceCliPath, @@ -101,6 +101,7 @@ export async function createPluginLocalRecipes( return result.capability; }, }); + const executor = new RecipeExecutor(); try { const local = await createLocalRecipeService({ cwd, @@ -179,7 +180,7 @@ export async function createPluginLocalRecipes( ), ); try { - return await executeRecipeProgram( + return await executor.execute( artifact, input, { @@ -244,14 +245,14 @@ export async function createPluginLocalRecipes( await local.close(); } finally { recording.clear(); - await models.dispose(); + await Promise.all([executor.close(), models.dispose()]); } })(); return closing; }, }; } catch (error) { - await models.dispose(); + await Promise.all([executor.close(), models.dispose()]); throw error; } } diff --git a/ts/packages/copilot-plugin/src/extension/recipe-native-host.ts b/ts/packages/copilot-plugin/src/extension/recipe-native-host.ts index 325d108ecd..7fd4b7d136 100644 --- a/ts/packages/copilot-plugin/src/extension/recipe-native-host.ts +++ b/ts/packages/copilot-plugin/src/extension/recipe-native-host.ts @@ -198,6 +198,9 @@ async function inspect( binding: RecipeBindingScope, step: Pick, policy: NativeRecipeHostPolicy, + metadata: () => ReturnType< + NativeRecipeSession["rpc"]["tools"]["getCurrentMetadata"] + > = () => session.rpc.tools.getCurrentMetadata(), ): Promise< | (NativeRecipeCapabilityInspection & { nativeName?: string }) | { status: "available"; capability: RecipeCapability; nativeName: string } @@ -218,7 +221,17 @@ async function inspect( await session.rpc.tools.initializeAndValidate(); // SDK 1.0.13 has no session.tools.list. This is its actual session-local // listing API; server.tools.list would omit MCP and extension schemas. - const { tools } = await session.rpc.tools.getCurrentMetadata(); + const { tools } = await metadata(); + if ( + !sameNativeRecipeBinding( + binding, + await captureNativeRecipeBinding(session, binding.profileId), + ) + ) + return { + status: "unsupported", + message: "Native binding changed during discovery.", + }; const matches = (tools ?? []).filter((tool) => step.mcpServerName ? tool.mcpServerName === step.mcpServerName && @@ -428,13 +441,49 @@ export function createNativeRecipeHost( ): RecipeHost { const captured = structuredClone(binding); const rules = structuredClone(policy); - const approved = new Map(); + const approved = new Map< + string, + { capability: RecipeCapability; nativeName: string } + >(); const current = () => captureNativeRecipeBinding(session, captured.profileId); + let inventory: + | ReturnType + | undefined; + const metadata = () => { + if (!inventory) { + const pending = session.rpc.tools + .getCurrentMetadata() + .then((value) => structuredClone(value)); + inventory = pending; + void pending.catch(() => { + if (inventory === pending) inventory = undefined; + }); + } + return inventory; + }; const host: RecipeHost = { async inspectCapability(requirement, scope) { - if (!sameNativeRecipeBinding(scope, captured)) + if ( + !sameNativeRecipeBinding(scope, captured) || + !sameNativeRecipeBinding(captured, await current()) + ) return { status: "unsupported", message: "Binding mismatch." }; + const previous = approved.get(requirement.id); + if (previous) { + if ( + checkRecipeCapability(requirement, previous.capability) + .length + ) + return { + status: "unsupported", + message: "Tool contract changed.", + }; + return { + status: "available", + capability: structuredClone(previous.capability), + }; + } const result = await inspect( session, captured, @@ -445,14 +494,23 @@ export function createNativeRecipeHost( : {}), }, rules, + metadata, ); if (result.status !== "available") return result; + if (!result.nativeName) + return { + status: "unsupported", + message: "Missing native tool identity.", + }; if (checkRecipeCapability(requirement, result.capability).length) return { status: "unsupported", message: "Tool contract changed.", }; - approved.set(requirement.id, structuredClone(result.capability)); + approved.set(requirement.id, { + capability: structuredClone(result.capability), + nativeName: result.nativeName, + }); return { status: "available", capability: result.capability }; }, async invokeCapability(capability, input, context) { @@ -461,31 +519,12 @@ export function createNativeRecipeHost( const preflight = approved.get(capability.id); if ( !preflight || - checkRecipeCapability(capability, preflight).length || - !sameNativeRecipeBinding(context.binding, captured) - ) - return failure("unsupported", "Missing exact bound preflight."); - const result = await inspect( - session, - captured, - { - toolName: capability.operation, - ...(capability.transport === "mcp" - ? { mcpServerName: capability.providerId } - : {}), - }, - rules, - ); - if ( - result.status !== "available" || - !result.nativeName || - checkRecipeCapability(capability, result.capability).length || + checkRecipeCapability(capability, preflight.capability) + .length || + !sameNativeRecipeBinding(context.binding, captured) || !sameNativeRecipeBinding(captured, await current()) ) - return failure( - "unsupported", - "Binding or tool contract changed.", - ); + return failure("unsupported", "Missing exact bound preflight."); if ( !structuredActionTool( { @@ -502,7 +541,7 @@ export function createNativeRecipeHost( ); const outcome = await boundedToolExecution( session, - result.nativeName, + preflight.nativeName, input, context, rules, diff --git a/ts/packages/copilot-plugin/test/macroAuthoringWorker.spec.ts b/ts/packages/copilot-plugin/test/macroAuthoringWorker.spec.ts index c34643a30a..e72ed2d02b 100644 --- a/ts/packages/copilot-plugin/test/macroAuthoringWorker.spec.ts +++ b/ts/packages/copilot-plugin/test/macroAuthoringWorker.spec.ts @@ -4,6 +4,7 @@ import { jest } from "@jest/globals"; import type { MacroAuthoringRequest, + MacroAuthoringClaim, MacroAuthoringSubmission, MacroLearningJob, MacroLearningMode, @@ -87,6 +88,12 @@ describe("durable Copilot authoring worker", () => { ) { let mode: MacroLearningMode = "all"; let status: MacroLearningJob["status"] = "awaitingAuthor"; + const notifications = new Set<() => void>(); + let disconnect: ((error: Error) => void) | undefined; + const notify = () => { + for (const resolve of notifications) resolve(); + notifications.clear(); + }; const job = (): MacroLearningJob => ({ jobId: "job", traceId: "capture", @@ -98,7 +105,20 @@ describe("durable Copilot authoring worker", () => { updatedAt: new Date().toISOString(), }); const connection = { - claimMacroAuthoringRequest: jest.fn(async () => requests.shift()), + claimMacroAuthoringRequest: jest.fn( + async (claim: MacroAuthoringClaim) => { + const request = claim.observeOnly + ? undefined + : requests.shift(); + if (request) return request; + return new Promise( + (resolve, reject) => { + notifications.add(() => resolve(undefined)); + disconnect = reject; + }, + ); + }, + ), submitMacroAuthoringResponse: jest.fn( async (_submission: MacroAuthoringSubmission) => {}, ), @@ -118,10 +138,11 @@ describe("durable Copilot authoring worker", () => { })), ); const report = jest.fn<(message: string) => void>(); + const connect = jest.fn(async () => connection); worker = new MacroAuthoringWorker({ cwd, sessionId: "host", - connect: async () => connection, + connect, launch: start, report, }); @@ -129,11 +150,17 @@ describe("durable Copilot authoring worker", () => { connection, start, report, + connect, + enqueue: (request: MacroAuthoringRequest) => requests.push(request), + notify, + disconnect: () => disconnect?.(new Error("IPC disconnected")), setOff: () => { mode = "off"; + notify(); }, cancel: () => { status = "cancelled"; + notify(); }, }; } @@ -245,4 +272,44 @@ describe("durable Copilot authoring worker", () => { }), ); }); + + it("does not poll a quiet owner every second and reconciles a wakeup against durable claims", async () => { + jest.useFakeTimers(); + const fixture = setup(undefined, []); + worker!.kick(); + await settle(); + await jest.advanceTimersByTimeAsync(20_000); + expect( + fixture.connection.claimMacroAuthoringRequest, + ).toHaveBeenCalledTimes(1); + expect( + fixture.connection.getMacroLearningPreference, + ).toHaveBeenCalledTimes(1); + fixture.enqueue(request("late")); + fixture.notify(); + await settle(); + expect(fixture.start).toHaveBeenCalledTimes(1); + expect( + fixture.connection.submitMacroAuthoringResponse, + ).toHaveBeenCalledWith(expect.objectContaining({ requestId: "late" })); + }); + + it("reconciles durable work at reconnect even when its notification was missed", async () => { + jest.useFakeTimers(); + const fixture = setup(undefined, []); + worker!.kick(); + await settle(); + fixture.disconnect(); + await settle(); + fixture.enqueue(request("missed")); + await jest.advanceTimersByTimeAsync(5_000); + await settle(); + expect(fixture.connect).toHaveBeenCalledTimes(2); + expect(fixture.start).toHaveBeenCalledTimes(1); + expect( + fixture.connection.submitMacroAuthoringResponse, + ).toHaveBeenCalledWith( + expect.objectContaining({ requestId: "missed" }), + ); + }); }); diff --git a/ts/packages/copilot-plugin/test/recipeNativeHost.spec.ts b/ts/packages/copilot-plugin/test/recipeNativeHost.spec.ts index abfffbeadf..b5c5986ed8 100644 --- a/ts/packages/copilot-plugin/test/recipeNativeHost.spec.ts +++ b/ts/packages/copilot-plugin/test/recipeNativeHost.spec.ts @@ -161,6 +161,7 @@ describe("native recipe host", () => { ).status, ).toBe("unsupported"); f.schema.required.push("new-field"); + f.emit("session.tools_updated"); expect( ( await f.host.invokeCapability( @@ -296,9 +297,9 @@ describe("native recipe host", () => { it("bounds inspection and prevents a late preflight from dispatching after cancellation", async () => { const f = await setup(); - const observed = deferred<{ tools: typeof f.tools }>(); + const observed = deferred(); const started = deferred(); - f.getCurrentMetadata.mockImplementation(() => { + f.snapshot.mockImplementation(() => { started.resolve(); return observed.promise; }); @@ -314,9 +315,112 @@ describe("native recipe host", () => { await started.promise; controller.abort(); expect((await call).status).toBe("execution_uncertain"); - observed.resolve({ tools: f.tools }); + observed.resolve(f.metadata); await new Promise((resolve) => setTimeout(resolve, 5)); expect(f.execute).not.toHaveBeenCalled(); expect(f.abort).not.toHaveBeenCalled(); }); + + it("reuses discovery within the bound host while retaining live binding checks", async () => { + const f = await setup(); + const discoveries = f.getCurrentMetadata.mock.calls.length; + const bindings = f.snapshot.mock.calls.length; + for (let i = 0; i < 3; i++) { + expect( + (await f.host.inspectCapability(f.capability, f.binding)) + .status, + ).toBe("available"); + expect( + ( + await f.host.invokeCapability( + f.capability, + { path: "x" }, + f.context, + ) + ).status, + ).toBe("completed"); + } + expect(f.getCurrentMetadata).toHaveBeenCalledTimes(discoveries); + expect(f.snapshot.mock.calls.length).toBeGreaterThan(bindings); + f.emit("session.permissions_changed"); + expect( + ( + await f.host.invokeCapability( + f.capability, + { path: "x" }, + f.context, + ) + ).status, + ).toBe("unsupported"); + expect(f.execute).toHaveBeenCalledTimes(3); + }); + + it("retries failed discovery instead of caching an unavailable inventory", async () => { + const f = await setup(); + const host = createNativeRecipeHost(f.session, f.binding, semantics); + f.getCurrentMetadata.mockRejectedValueOnce( + new Error("Discovery failed"), + ); + await expect( + host.inspectCapability(f.capability, f.binding), + ).resolves.toEqual({ + status: "unavailable", + message: "Error: Discovery failed", + }); + expect( + (await host.inspectCapability(f.capability, f.binding)).status, + ).toBe("available"); + expect( + ( + await host.invokeCapability( + f.capability, + { path: "x" }, + f.context, + ) + ).status, + ).toBe("completed"); + expect(f.execute).toHaveBeenCalledTimes(1); + }); + + it("keeps returned descriptors separate from the approved contract", async () => { + const f = await setup(); + const inspection = await f.host.inspectCapability( + f.capability, + f.binding, + ); + expect(inspection.status).toBe("available"); + if (inspection.status !== "available") + throw new Error("Expected preflight"); + inspection.capability.operation = "different"; + expect( + ( + await f.host.invokeCapability( + f.capability, + { path: "x" }, + f.context, + ) + ).status, + ).toBe("completed"); + }); + + it("rejects inventory captured across a generation change", async () => { + const f = await setup(); + const host = createNativeRecipeHost(f.session, f.binding, semantics); + const observed = deferred<{ tools: typeof f.tools }>(); + f.getCurrentMetadata.mockImplementation(() => observed.promise); + const inspection = host.inspectCapability(f.capability, f.binding); + f.emit("session.tools_updated"); + observed.resolve({ tools: f.tools }); + expect((await inspection).status).toBe("unsupported"); + expect( + ( + await host.invokeCapability( + f.capability, + { path: "x" }, + f.context, + ) + ).status, + ).toBe("unsupported"); + expect(f.execute).not.toHaveBeenCalled(); + }); }); diff --git a/ts/packages/knowPro/src/index.ts b/ts/packages/knowPro/src/index.ts index 36deb2297f..f2b24f7334 100644 --- a/ts/packages/knowPro/src/index.ts +++ b/ts/packages/knowPro/src/index.ts @@ -42,3 +42,4 @@ export * from "./storage.js"; export * as dataFrame from "./dataFrame/index.js"; export { createConversationFromData } from "./common.js"; +export * from "./segmentedJson.js"; diff --git a/ts/packages/knowPro/src/search.ts b/ts/packages/knowPro/src/search.ts index 186fd9af92..e69b2e0452 100644 --- a/ts/packages/knowPro/src/search.ts +++ b/ts/packages/knowPro/src/search.ts @@ -56,6 +56,8 @@ export type SearchQueryExpr = { }; export interface SearchOptions { + /** Applied to both knowledge and vector candidates before ranking limits. */ + messageRangesInScope?: import("./interfaces.js").TextRange[]; maxKnowledgeMatches?: number | undefined; exactMatch?: boolean | undefined; maxMessageMatches?: number | undefined; @@ -465,7 +467,11 @@ class QueryCompiler { ): Promise { const messageIndex = this.conversation.secondaryIndexes?.messageIndex; if (messageIndex !== undefined) { - const scopeExpr = await this.compileScope(undefined, whenFilter); + const scopeExpr = await this.compileScope( + undefined, + whenFilter, + options, + ); return this.compileMessageSimilarity(query, scopeExpr, options); } return undefined; @@ -478,7 +484,7 @@ class QueryCompiler { ): Promise>> { let selectExpr = this.compileSelect( searchTermGroup, - await this.compileScope(searchTermGroup, filter), + await this.compileScope(searchTermGroup, filter, options), options, ); // Constrain the select with scopes and 'where' @@ -630,11 +636,16 @@ class QueryCompiler { private async compileScope( termGroup?: SearchTermGroup, filter?: WhenFilter | undefined, + options?: SearchOptions, ): Promise { - let scopeSelectors: q.IQueryTextRangeSelector[] | undefined; + const scopeSelectors: q.IQueryTextRangeSelector[] = []; + if (options?.messageRangesInScope !== undefined) { + scopeSelectors.push( + new q.TextRangeSelector(options.messageRangesInScope), + ); + } // First, use any provided date ranges to select scope if (filter && filter.dateRange) { - scopeSelectors ??= []; scopeSelectors.push( new q.TextRangesInDateRangeSelector(filter.dateRange), ); @@ -644,7 +655,6 @@ class QueryCompiler { // // If specific scoping terms were provided if (filter && filter.scopeDefiningTerms !== undefined) { - scopeSelectors ??= []; this.addTermsScopeSelector( filter.scopeDefiningTerms, scopeSelectors, @@ -654,20 +664,17 @@ class QueryCompiler { const actionTermsGroup = this.getActionTermsFromSearchGroup(termGroup); if (actionTermsGroup !== undefined) { - scopeSelectors ??= []; this.addTermsScopeSelector(actionTermsGroup, scopeSelectors); } } // Include any ranges directly provided by the caller if (filter && filter.textRangesInScope) { - scopeSelectors ??= []; - scopeSelectors?.push( + scopeSelectors.push( new q.TextRangeSelector(filter.textRangesInScope), ); } // Tags... if (filter && filter.tags && filter.tags.length > 0) { - scopeSelectors ??= []; this.addTermsScopeSelector( createTagSearchTermGroup(filter.tags, true), scopeSelectors, @@ -680,7 +687,6 @@ class QueryCompiler { filter.threadDescription, ); if (threadsInScope) { - scopeSelectors ??= []; scopeSelectors.push( new q.ThreadSelector( threadsInScope.map( @@ -696,11 +702,10 @@ class QueryCompiler { filter.tagMatchingTerms && filter.tagMatchingTerms.terms.length > 0 ) { - scopeSelectors ??= []; this.addSTagScopeSelector(filter.tagMatchingTerms, scopeSelectors); } - return scopeSelectors && scopeSelectors.length > 0 + return scopeSelectors.length > 0 ? new q.GetScopeExpr(scopeSelectors) : undefined; } diff --git a/ts/packages/knowPro/src/segmentedJson.ts b/ts/packages/knowPro/src/segmentedJson.ts new file mode 100644 index 0000000000..7f22433083 --- /dev/null +++ b/ts/packages/knowPro/src/segmentedJson.ts @@ -0,0 +1,330 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID } from "node:crypto"; +import { + lstat, + mkdir, + readFile, + readdir, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +export const jsonSegmentDirectory = ".segments"; +const digestPattern = /^[a-f0-9]{64}$/; +const verified = new Map>(); +const tails = new Map>(); + +async function serialize( + root: string, + operation: () => Promise, +): Promise { + const result = (tails.get(root) ?? Promise.resolve()).then(operation); + const tail = result.then( + () => {}, + () => {}, + ); + tails.set(root, tail); + try { + return await result; + } finally { + if (tails.get(root) === tail) tails.delete(root); + } +} +type JsonNode = + | { kind: "value"; value: unknown } + | { kind: "items" | "chunks"; children: string[] } + | { kind: "object"; entries: [string, string][] }; + +export interface SegmentedJsonDescriptor { + segmentedJsonVersion: 1; + root: string; + metadata: Record; +} + +function directory(file: string): string { + return path.join(path.dirname(path.dirname(file)), jsonSegmentDirectory); +} + +function digest(bytes: string | Buffer): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function isSegmentedJson( + value: unknown, +): value is SegmentedJsonDescriptor { + if ( + !value || + typeof value !== "object" || + !("segmentedJsonVersion" in value) + ) + return false; + if ( + value.segmentedJsonVersion !== 1 || + !("root" in value) || + typeof value.root !== "string" || + !digestPattern.test(value.root) || + !("metadata" in value) || + !value.metadata || + typeof value.metadata !== "object" || + Object.values(value.metadata).some( + (item) => !Number.isSafeInteger(item) || item < 0, + ) + ) + throw new Error( + "Invalid or unsupported segmented JSON descriptor; rebuild the derived index", + ); + return true; +} + +async function readNode(root: string, hash: string): Promise { + if (!digestPattern.test(hash)) + throw new Error("Unsafe JSON segment reference"); + const file = path.join(root, `${hash}.json`); + const info = await lstat(file); + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) + throw new Error(`Unsafe JSON segment '${file}'`); + const bytes = await readFile(file); + if (digest(bytes) !== hash) + throw new Error(`Corrupt JSON segment '${file}'`); + const node: JsonNode = JSON.parse(bytes.toString("utf8")); + if (!node || !["value", "items", "chunks", "object"].includes(node.kind)) + throw new Error(`Invalid JSON segment '${file}'`); + const known = verified.get(root) ?? new Set(); + known.add(hash); + verified.set(root, known); + return node; +} + +async function hydrate( + root: string, + hash: string, + ancestors = new Set(), +): Promise { + if (ancestors.has(hash)) throw new Error("Cyclic JSON segment reference"); + const next = new Set(ancestors).add(hash); + const node = await readNode(root, hash); + switch (node.kind) { + case "value": + return node.value; + case "object": { + const value: Record = {}; + for (const [key, child] of node.entries) { + Object.defineProperty(value, key, { + value: await hydrate(root, child, next), + enumerable: true, + writable: true, + configurable: true, + }); + } + return value; + } + default: { + const values: unknown[] = []; + for (const child of node.children) { + const value = await hydrate(root, child, next); + if (node.kind === "chunks") { + if (!Array.isArray(value)) + throw new Error("Invalid JSON array segment"); + values.push(...value); + } else values.push(value); + } + return values; + } + } +} + +export async function readSegmentedJson( + file: string, + descriptor: SegmentedJsonDescriptor, +): Promise { + return serialize(directory(file), () => + hydrate(directory(file), descriptor.root), + ); +} + +async function publish(file: string, bytes: string): Promise { + const staging = `${file}.${randomUUID()}.staging`; + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(staging, bytes, { flag: "wx" }); + try { + for (let attempt = 0; ; attempt++) { + try { + await rename(staging, file); + return; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if ( + process.platform !== "win32" || + !["EPERM", "EACCES", "EBUSY"].includes(code ?? "") || + attempt >= 79 + ) + throw error; + await delay(25); + } + } + } finally { + await rm(staging, { force: true }); + } +} + +async function writeSegments( + file: string, + value: unknown, + metadata: Record, + checkpoint?: ( + point: "segment-written" | "descriptor-prepared", + ) => Promise, +): Promise { + const root = directory(file); + await mkdir(root, { recursive: true }); + const existing = new Set(await readdir(root)); + const known = verified.get(root) ?? new Set(); + verified.set(root, known); + const save = async (item: unknown): Promise => { + const json = JSON.stringify(item); + if (json === undefined) + throw new Error("Segmented JSON requires a serializable value"); + let node: JsonNode; + if ( + Buffer.byteLength(json) <= 16_384 || + !item || + typeof item !== "object" + ) + node = { kind: "value", value: item }; + else if (Array.isArray(item)) { + const children: string[] = []; + if (item.length > 64) { + for (let i = 0; i < item.length; i += 64) + children.push(await save(item.slice(i, i + 64))); + node = { kind: "chunks", children }; + } else { + for (const entry of item) + children.push( + await save(entry === undefined ? null : entry), + ); + node = { kind: "items", children }; + } + } else { + const entries: [string, string][] = []; + for (const [key, entry] of Object.entries(item)) { + if (entry !== undefined) entries.push([key, await save(entry)]); + } + node = { kind: "object", entries }; + } + const bytes = `${JSON.stringify(node)}\n`; + const hash = digest(bytes); + const name = `${hash}.json`; + if (!existing.has(name)) { + try { + await writeFile(path.join(root, name), bytes, { flag: "wx" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") + throw error; + await readNode(root, hash); + } + existing.add(name); + known.add(hash); + await checkpoint?.("segment-written"); + } else if (!known.has(hash)) await readNode(root, hash); + return hash; + }; + const hash = await save(value); + await checkpoint?.("descriptor-prepared"); + await publish( + file, + `${JSON.stringify({ + segmentedJsonVersion: 1, + root: hash, + metadata, + } satisfies SegmentedJsonDescriptor)}\n`, + ); +} + +export async function writeSegmentedJson( + file: string, + value: unknown, + metadata: Record, + checkpoint?: ( + point: "segment-written" | "descriptor-prepared", + ) => Promise, +): Promise { + return serialize(directory(file), () => + writeSegments(file, value, metadata, checkpoint), + ); +} + +async function collectSegments( + generationRoot: string, + baseFileName = "corpus_data.json", +): Promise { + const root = path.join(generationRoot, jsonSegmentDirectory); + let entries; + try { + entries = await readdir(generationRoot, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + + const retained = new Set(); + const visit = async (hash: string): Promise => { + if (retained.has(hash)) return; + const node = await readNode(root, hash); + retained.add(hash); + const children = + node.kind === "object" + ? node.entries.map(([, child]) => child) + : node.kind === "value" + ? [] + : node.children; + for (const child of children) await visit(child); + }; + for (const entry of entries) { + if (!entry.isDirectory() || entry.name === jsonSegmentDirectory) + continue; + let value: unknown; + try { + value = JSON.parse( + await readFile( + path.join(generationRoot, entry.name, baseFileName), + "utf8", + ), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (isSegmentedJson(value)) await visit(value.root); + } + let files: string[]; + try { + files = await readdir(root); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + for (const name of files) { + const hash = name.replace(/\.json$/, ""); + if (!digestPattern.test(hash)) + throw new Error(`Unexpected JSON segment '${name}'`); + if (!retained.has(hash)) { + await rm(path.join(root, name)); + verified.get(root)?.delete(hash); + } + } +} + +/** Generation descriptors pin their immutable nodes; concurrent writers queue behind collection. */ +export async function collectJsonSegments( + generationRoot: string, + baseFileName = "corpus_data.json", +): Promise { + return serialize(path.join(generationRoot, jsonSegmentDirectory), () => + collectSegments(generationRoot, baseFileName), + ); +} diff --git a/ts/packages/knowPro/src/serialization.ts b/ts/packages/knowPro/src/serialization.ts index b1fb051195..9dc5f26c11 100644 --- a/ts/packages/knowPro/src/serialization.ts +++ b/ts/packages/knowPro/src/serialization.ts @@ -10,6 +10,11 @@ import { } from "@typeagent/agent-runtime"; import { deserializeEmbeddings, serializeEmbeddings } from "./fuzzyIndex.js"; import path from "path"; +import { + isSegmentedJson, + readSegmentedJson, + writeSegmentedJson, +} from "./segmentedJson.js"; import { IConversationDataWithIndexes } from "./secondaryIndexes.js"; import { EmbeddingModelMetadata, @@ -33,6 +38,7 @@ export async function writeConversationDataToFile( baseFileName: string, modelMeta?: EmbeddingModelMetadata, identities?: PersistedEmbeddingIdentities, + segmented = false, ): Promise { identities ??= (conversationData as EmbeddingIdentifiedConversationData) .embeddingIdentities; @@ -42,6 +48,29 @@ export async function writeConversationDataToFile( validateIdentityVectors(fileData, identities); fileData.jsonData.embeddingFileHeader!.identities = identities; } + if (segmented) { + const relatedCount = + fileData.jsonData.embeddingFileHeader?.relatedCount ?? 0; + const vectors = + fileData.binaryData.embeddings?.map((vector) => + Array.from(vector), + ) ?? []; + await writeSegmentedJson( + path.join(dirPath, baseFileName + DataFileSuffix), + { + jsonData: fileData.jsonData, + binaryData: { + relatedEmbeddings: vectors.slice(0, relatedCount), + messageEmbeddings: vectors.slice(relatedCount), + }, + }, + { + messages: conversationData.messages.length, + semanticRefs: conversationData.semanticRefs?.length ?? 0, + }, + ); + return; + } if (fileData.binaryData) { if ( fileData.binaryData.embeddings && @@ -75,10 +104,18 @@ export async function readConversationDataFromFile( if (!jsonData) { return undefined; } - const fileData: ConversationFileData = { + let fileData: ConversationFileData = { jsonData: jsonData, binaryData: {}, }; + if (isSegmentedJson(jsonData)) { + fileData = segmentedFileData( + await readSegmentedJson( + path.join(dirPath, baseFileName + DataFileSuffix), + jsonData, + ), + ); + } validateFileData(fileData, embeddingSize); if (options?.loadEmbeddings === false) { dropPersistedEmbeddings(fileData.jsonData); @@ -101,12 +138,17 @@ export async function readConversationDataFromFile( dropPersistedEmbeddings(fileData.jsonData); } if (embeddingsCompatible && embeddingSize && embeddingSize > 0) { - const embeddingsBuffer = await readFile( - path.join(dirPath, baseFileName + EmbeddingFileSuffix), - ); - if (embeddingsBuffer) { - embeddings = deserializeEmbeddings(embeddingsBuffer, embeddingSize); - fileData.binaryData.embeddings = embeddings; + if (!fileData.binaryData.embeddings) { + const embeddingsBuffer = await readFile( + path.join(dirPath, baseFileName + EmbeddingFileSuffix), + ); + if (embeddingsBuffer) { + embeddings = deserializeEmbeddings( + embeddingsBuffer, + embeddingSize, + ); + fileData.binaryData.embeddings = embeddings; + } } } fileData.jsonData.fileHeader ??= createFileHeader(); @@ -115,6 +157,49 @@ export async function readConversationDataFromFile( return fromConversationFileData(fileData); } +function readSegmentVectors(value: unknown): Float32Array[] { + if ( + !Array.isArray(value) || + value.some( + (vector) => + !Array.isArray(vector) || + vector.some( + (number) => + typeof number !== "number" || !Number.isFinite(number), + ), + ) + ) + throw new Error("Invalid segmented conversation vectors"); + return value.map((vector: number[]) => new Float32Array(vector)); +} + +function segmentedFileData(value: unknown): ConversationFileData { + if ( + !value || + typeof value !== "object" || + !("jsonData" in value) || + !value.jsonData || + typeof value.jsonData !== "object" || + !("messages" in value.jsonData) || + !Array.isArray(value.jsonData.messages) || + !("binaryData" in value) || + !value.binaryData || + typeof value.binaryData !== "object" || + !("relatedEmbeddings" in value.binaryData) || + !("messageEmbeddings" in value.binaryData) + ) + throw new Error("Invalid segmented conversation payload"); + return { + jsonData: value.jsonData as ConversationJsonData, + binaryData: { + embeddings: [ + ...readSegmentVectors(value.binaryData.relatedEmbeddings), + ...readSegmentVectors(value.binaryData.messageEmbeddings), + ], + }, + }; +} + export async function readConversationDataFromBuffer( jsonData: string, embeddingsBuffer: Buffer, diff --git a/ts/packages/knowPro/test/embeddingRoles.spec.ts b/ts/packages/knowPro/test/embeddingRoles.spec.ts index 38ba97b778..830b4310e2 100644 --- a/ts/packages/knowPro/test/embeddingRoles.spec.ts +++ b/ts/packages/knowPro/test/embeddingRoles.spec.ts @@ -16,6 +16,11 @@ import { import { MessageTextIndex } from "../src/messageIndex.js"; import { TermEmbeddingIndex } from "../src/relatedTermsIndex.js"; import { TextToTextLocationIndex } from "../src/textLocationIndex.js"; +import { searchConversationByTextSimilarity } from "../src/search.js"; +import { IConversation, TextRange } from "../src/interfaces.js"; +import { createTestMessages } from "./testMessage.js"; +import { SemanticRefCollection } from "../src/storage.js"; +import { TermToSemanticRefIndex } from "../src/conversationIndex.js"; describe("KnowPro embedding roles (synthetic/offline)", () => { let calls: string[]; @@ -111,6 +116,85 @@ describe("KnowPro embedding roles (synthetic/offline)", () => { expect(calls).toEqual(["query:search"]); }); + test("revision eligibility scopes vector candidates before TopN and intersects caller filters", async () => { + const index = new MessageTextIndex({ + embeddingIndexSettings: createTextEmbeddingIndexSettings( + model(), + 2, + ), + }); + index.deserialize({ + messageCount: 4, + indexData: { + embeddings: [ + [1, 0], + [0.8, 0.6], + [1, 0], + [0.6, 0.8], + ].map((vector) => new Float32Array(vector)), + textLocations: [0, 1, 2, 3].map((messageOrdinal) => ({ + messageOrdinal, + chunkOrdinal: 0, + })), + }, + }); + const conversation: IConversation = { + nameTag: "eligibility", + tags: [], + messages: createTestMessages(4), + semanticRefs: new SemanticRefCollection(), + semanticRefIndex: new TermToSemanticRefIndex(), + secondaryIndexes: { messageIndex: index }, + }; + const range = (messageOrdinal: number): TextRange => ({ + start: { messageOrdinal, chunkOrdinal: 0 }, + end: { messageOrdinal: messageOrdinal + 1, chunkOrdinal: 0 }, + }); + const ranges = [range(1), range(3)]; + const expected = index + .lookupByEmbedding(new Float32Array([1, 0]), undefined, 0) + .filter((match) => match.messageOrdinal % 2 === 1); + for (const limit of [1, 2, 10]) { + const result = await searchConversationByTextSimilarity( + conversation, + "query", + undefined, + { + messageRangesInScope: ranges, + maxMessageMatches: limit, + thresholdScore: 0, + }, + ); + expect(result?.messageMatches).toEqual(expected.slice(0, limit)); + } + const scoped = await searchConversationByTextSimilarity( + conversation, + "query", + { + textRangesInScope: [range(3)], + }, + { + messageRangesInScope: ranges, + maxMessageMatches: 1, + thresholdScore: 0, + }, + ); + expect(scoped?.messageMatches).toEqual( + expected.filter((match) => match.messageOrdinal === 3), + ); + const empty = await searchConversationByTextSimilarity( + conversation, + "query", + undefined, + { + messageRangesInScope: [], + maxMessageMatches: 1, + thresholdScore: 0, + }, + ); + expect(empty?.messageMatches).toEqual([]); + }); + test("related terms use symmetric documents and never answer query-role cache keys", async () => { const index = new TermEmbeddingIndex( createTextEmbeddingIndexSettings(model(), 2), diff --git a/ts/packages/knowPro/test/segmentedJson.spec.ts b/ts/packages/knowPro/test/segmentedJson.spec.ts new file mode 100644 index 0000000000..f5b9c381c1 --- /dev/null +++ b/ts/packages/knowPro/test/segmentedJson.spec.ts @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + mkdtemp, + mkdir, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { + collectJsonSegments, + isSegmentedJson, + readSegmentedJson, + writeSegmentedJson, +} from "../src/segmentedJson.js"; + +describe("immutable segmented JSON publication", () => { + let root: string; + let file: string; + const records = () => + Array.from({ length: 1024 }, (_, ordinal) => ({ + ordinal, + text: `${"Retained source knowledge. ".repeat(24)}${ordinal}`, + })); + beforeEach(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "knowpro-segments-")); + file = path.join(root, "generation", "corpus_data.json"); + }); + afterEach(async () => { + await rm(root, { recursive: true, force: true }); + }); + const descriptor = async (target: string) => { + const value: unknown = JSON.parse(await readFile(target, "utf8")); + if (!isSegmentedJson(value)) + throw new Error("Missing segment descriptor"); + return value; + }; + + test("append writes only changed nodes and retained values, vectors and ordinals roundtrip exactly", async () => { + const data = { + records: records(), + vectors: [[0.6, 0.8]], + optional: undefined, + }; + await writeSegmentedJson(file, data, { messages: 1024 }); + const blobs = path.join(root, ".segments"); + const original = new Map( + await Promise.all( + (await readdir(blobs)).map( + async (name) => + [name, await stat(path.join(blobs, name))] as const, + ), + ), + ); + const next = path.join(root, "next-generation", "corpus_data.json"); + data.records.push({ ordinal: 1024, text: "Only this source is new" }); + await writeSegmentedJson(next, data, { messages: 1025 }); + let changedBytes = 0; + for (const name of await readdir(blobs)) { + const info = await stat(path.join(blobs, name)); + if (original.has(name)) { + expect(info.ino).toBe(original.get(name)!.ino); + expect(info.mtimeMs).toBe(original.get(name)!.mtimeMs); + } else changedBytes += info.size; + } + expect(changedBytes).toBeLessThan(20_000); + expect(changedBytes).toBeLessThan( + Buffer.byteLength(JSON.stringify(data)) / 20, + ); + expect(await readSegmentedJson(next, await descriptor(next))).toEqual( + JSON.parse(JSON.stringify(data)), + ); + expect( + ( + (await readSegmentedJson(file, await descriptor(file))) as { + records: unknown[]; + } + ).records, + ).toHaveLength(1024); + await collectJsonSegments(root); + expect(await readSegmentedJson(next, await descriptor(next))).toEqual( + JSON.parse(JSON.stringify(data)), + ); + }); + + test.each(["segment-written", "descriptor-prepared"] as const)( + "a %s failure retains the previous publication and orphan collection is recoverable", + async (point) => { + const initial = { records: records() }; + await writeSegmentedJson(file, initial, {}); + const before = await readFile(file, "utf8"); + await expect( + writeSegmentedJson( + file, + { + records: [ + ...initial.records, + { text: "Uncommitted private payload" }, + ], + }, + {}, + async (actual) => { + if (actual === point) + throw new Error("Expected interruption"); + }, + ), + ).rejects.toThrow("Expected interruption"); + expect(await readFile(file, "utf8")).toBe(before); + await collectJsonSegments(root); + expect( + await readSegmentedJson(file, await descriptor(file)), + ).toEqual(initial); + for (const name of await readdir(path.join(root, ".segments"))) + expect( + await readFile(path.join(root, ".segments", name), "utf8"), + ).not.toContain("Uncommitted private payload"); + }, + ); + + test("collection removes forgotten bytes only after obsolete generation descriptors disappear", async () => { + await writeSegmentedJson( + file, + { text: "Private forgotten payload" }, + {}, + ); + const next = path.join(root, "next", "corpus_data.json"); + await writeSegmentedJson(next, { text: "Retained public payload" }, {}); + await collectJsonSegments(root); + expect(await readSegmentedJson(file, await descriptor(file))).toEqual({ + text: "Private forgotten payload", + }); + await rm(path.dirname(file), { recursive: true }); + await collectJsonSegments(root); + const bytes = await Promise.all( + (await readdir(path.join(root, ".segments"))).map((name) => + readFile(path.join(root, ".segments", name), "utf8"), + ), + ); + expect(bytes.join("\n")).not.toContain("Private forgotten payload"); + expect(await readSegmentedJson(next, await descriptor(next))).toEqual({ + text: "Retained public payload", + }); + }); + + test("referenced corruption and unsupported or unsafe descriptors fail explicitly", async () => { + await writeSegmentedJson(file, { text: "Verified" }, {}); + const value = await descriptor(file); + await writeFile( + path.join(root, ".segments", `${value.root}.json`), + "{}", + ); + await expect(readSegmentedJson(file, value)).rejects.toThrow("Corrupt"); + expect(() => + isSegmentedJson({ ...value, segmentedJsonVersion: 2 }), + ).toThrow("unsupported"); + expect(() => + isSegmentedJson({ ...value, root: "..\\outside" }), + ).toThrow("unsupported"); + }); + + test("concurrent publication and collection keep every published root readable", async () => { + const next = path.join(root, "next", "corpus_data.json"); + await mkdir(path.dirname(next), { recursive: true }); + await Promise.all([ + writeSegmentedJson(file, { records: records() }, {}), + writeSegmentedJson(next, { records: records().slice(0, 512) }, {}), + collectJsonSegments(root), + ]); + expect(await readSegmentedJson(file, await descriptor(file))).toEqual({ + records: records(), + }); + expect(await readSegmentedJson(next, await descriptor(next))).toEqual({ + records: records().slice(0, 512), + }); + }); +}); diff --git a/ts/packages/knowPro/test/serializationIdentity.spec.ts b/ts/packages/knowPro/test/serializationIdentity.spec.ts index 4a6ae193f5..bb7269e131 100644 --- a/ts/packages/knowPro/test/serializationIdentity.spec.ts +++ b/ts/packages/knowPro/test/serializationIdentity.spec.ts @@ -65,6 +65,43 @@ describe("identity-aware conversation serialization", () => { new Float32Array([1, 0]), ); }); + test("segmented JSON retains vector identities and exact float32 data without a monolithic binary rewrite", async () => { + const directory = path.join(root, "segmented"); + await writeConversationDataToFile( + data(), + directory, + "known", + { embeddingSize: 2 }, + { message: identity }, + true, + ); + expect(await fs.readdir(directory)).not.toContain( + "known_embeddings.bin", + ); + const loaded = await readConversationDataFromFile( + directory, + "known", + 2, + { message: identity }, + ); + expect(loaded!.messageIndexData!.indexData!.embeddings[0]).toEqual( + new Float32Array([1, 0]), + ); + await expect( + readConversationDataFromFile(directory, "known", 2, { + message: { ...identity, revision: "wrong" }, + }), + ).rejects.toBeInstanceOf(PersistedEmbeddingIdentityError); + const text = await readConversationDataFromFile( + directory, + "known", + 2, + undefined, + { loadEmbeddings: false }, + ); + expect(text!.messageIndexData!.indexData).toBeUndefined(); + expect(text!.messages).toEqual(data().messages); + }); test("strict reads reject missing vector artifact", async () => { await writeConversationDataToFile( data(), diff --git a/ts/packages/memory/conversation/src/common.ts b/ts/packages/memory/conversation/src/common.ts index a273ef9e8a..28e309a944 100644 --- a/ts/packages/memory/conversation/src/common.ts +++ b/ts/packages/memory/conversation/src/common.ts @@ -109,6 +109,7 @@ export async function writeMemoryDataToFile( settings: kp.ConversationSettings, dirPath: string, baseFileName: string, + segmented = false, ): Promise { const identities = await prepareConversationEmbeddings(settings); await kp.writeConversationDataToFile( @@ -117,6 +118,7 @@ export async function writeMemoryDataToFile( baseFileName, undefined, Object.keys(identities).length > 0 ? identities : undefined, + segmented, ); } diff --git a/ts/packages/memory/conversation/src/docMemory.ts b/ts/packages/memory/conversation/src/docMemory.ts index 1de552952a..6550073255 100644 --- a/ts/packages/memory/conversation/src/docMemory.ts +++ b/ts/packages/memory/conversation/src/docMemory.ts @@ -412,6 +412,7 @@ export class DocMemory public async writeToFile( dirPath: string, baseFileName: string, + segmented = false, ): Promise { await this.prepareEmbeddings(); const data = await this.serialize(); @@ -420,6 +421,7 @@ export class DocMemory this.settings.conversationSettings, dirPath, baseFileName, + segmented, ); } diff --git a/ts/packages/memory/service/README.md b/ts/packages/memory/service/README.md index cc8f820f79..73fe4a1834 100644 --- a/ts/packages/memory/service/README.md +++ b/ts/packages/memory/service/README.md @@ -68,6 +68,86 @@ Restart and reconciliation resume enrollment without an expiring preview token. Privacy removal outranks human edit protection and cancels late model results. The preview token alone does not deny access or constitute durable removal. +## Canonical revision text storage + +Corpus manifests use `storageVersion: 1`. Revision metadata contains a `text` +reference with the SHA-256 digest and byte length of an immutable corpus-local +`revision-text/.json` blob, not an inline `content` field. Each blob is a +UTF-8 JSON string; escaping preserves exact UTF-16 content, including lone +surrogates. Source hashes, citation coordinates and revision identities are +unchanged. Text remains available in the owner's in-memory source snapshots. + +New blobs are staged, read back and digest/size verified before manifest +publication. Existing immutable revision references are reused under the store's +single-owner lock. Manifests publish using staged atomic replacement with bounded +Windows retry, without moving the previous canonical file away first. This is a +process-restart recovery protocol, not a multi-file transaction or proof of +power-loss durability. Metadata serialization, source-count-dependent work and +index-generation copies/rebuilds remain. + +Only committed manifest references retain blobs; there is no separately committed +reference-count database. Replacement collects unreferenced text after commit, +and startup collects abandoned blobs/staging files. Missing, corrupt or unsafe +referenced blobs fail explicitly; derived indexes never reconstruct canonical +text. Content is shared only within a corpus and only while another retained +source/revision legitimately references it. + +Source forget physically purges last-reference text before clearing its existing +durable removal intent. Corpus clear similarly records `pendingCorpusClear`, +withholds corpus content reads/writes on cleanup failure, and resumes its +event/index/asset/text cleanup at restart before reopening reads. Text-reference +caches contain metadata only and evict removed references before deleting files. + +Older/unrecognized canonical manifest formats are rejected with explicit +reset/reimport guidance, never silently migrated or deleted. Preserve the store +and use its matching build to export needed data before explicitly resetting and +reimporting. Existing derived-index rebuild rules are unchanged. + +## Incremental indexes and event segments + +KnowPro-backed derived generations now publish a small `corpus_data.json` +descriptor over immutable content-addressed JSON nodes in the index root's +`.segments` directory. Generation copies copy pointers and generation metadata, +not retained node payloads. Related-term and message vectors have separate JSON +lanes and reconstruct the same Float32Array values. Other KnowPro clients keep +the existing serialization unless they explicitly opt in. + +An existing compatible source using `retainRevisionHistory`, without acquired +assets, can replace its active revision by indexing only changed content. +`eligibility.json` selects one retained revision per source. Message ordinals +remain stable; knowledge, fragments, subjects and vector candidates exclude +inactive revisions before ranking limits. Reactivating indexed historical +content requires no provider extraction. Replacement without history and +unsupported/custom indexes still rebuild, preserving physical removal behavior. +Explicit reindexing compacts inactive indexed revisions from current canonical +sources. Metadata serialization, full index hydration, in-memory JSON traversal, +linear vector search and retained-history growth are not eliminated. + +Canonical `events.jsonl` is now a version-1 publication descriptor, not a +concatenated log. Immutable JSONL event files live in `event-segments`; a +filename-derived lookup selects individual event bodies without decoding all +retained events. The pointer records monotonic sequence and independent +eligibility revision; suppression/tombstone metadata lives in a referenced blob. +Append publishes one event and a bounded pointer. Listing and search still load +the required event set, and cold lookup scans filenames. Same-eligibility event +index reconciliation extracts only the appended suffix. Deletion invalidates +derived generations even when later append restores the prior event count. + +Forgetting retains a durable pending-purge descriptor until event payloads, +obsolete suppression blobs and dependent indexes are physically removed. +Recovery finishes that purge before reads resume. Generation descriptors retain +shared JSON nodes; ordinary obsolete-node collection and explicit reindexing +remove unreferenced data, while privacy cleanup drains writers and removes +affected bytes before reopening reads. Canonical old/future event formats fail +with explicit reset/reimport guidance; no store is automatically migrated or +reset. These are process-restart recovery guarantees, not a power-loss +transaction across files. + +Single-document cancellation, like shared-batch cancellation, remains +`cancelling` until actual provider writes drain. An abort-ignoring provider can +delay terminal cancellation, close or privacy cleanup; no terminal success is +reported while it could restore removed bytes. + ## Automatic editorial maintenance Wiki and Projects revise complete small targets through existing family jobs. @@ -305,6 +385,27 @@ ingestion requests, with each document omitting `corpusId`. Acquisition remains host-owned. Limits are **50 documents / 8 MiB encoded request**, two active members per batch, at most four active batches; corpus commits retain their existing serialization. + +Eligible new-source members share a detached index generation in groups of at +most **two**, with a **25 ms admission window** starting when the first job is +durably accepted. This is not a provider-duration or queue-wait bound. An empty +corpus rebuilds the pair together; a nonempty corpus copies its compatible +generation once and appends both documents. Each member still has its own durable +job, progress, cancellation and retry identity. Ordinary standalone ingestion is +unchanged. Replacements, duplicate source IDs, revision guards, assets, +unsupported append generations and budgeted rehearsals keep the singleton path. + +Shared provider/publication failures fail the affected jobs explicitly. Before +publication, cancellation or a generation conflict discards the candidate and +retries unaffected members through singleton ingestion; corpus clear cancels +captured pre-clear admissions instead. Cleanup drains the actual provider +promise, including providers that ignore abort, before removing private files. +Committed sources retain the existing post-commit warning semantics. +Physical batch reads, staged publications and purge deletion share a per-batch +owner-local gate so rapid durable polling cannot hold a Windows file handle +across replacement. Reads still come from disk, and publication errors remain +errors rather than returning uncommitted in-memory state. + `measureBatchImportBytes(request)` returns the exact UTF-8 byte length of the durable JSON encoding, including `Uint8Array` assets encoded as `{ batchAssetBytes: "" }`. `assertBatchImportRequest(request)` applies @@ -910,6 +1011,13 @@ snapshot. Per-revision indexes remain outside Git under Source replacement marks dependent drafts stale without modifying historical revisions. Forget/clear quarantines history, creates a sanitized baseline, replaces the entire old Git object database (including unreachable blobs), and removes affected View search indexes before reopening reads. Interrupted privacy cleanup rejects reads until restart recovery finishes. No procedure state, settings or index is persisted or migrated by this service. +Git commits also retain full-fidelity per-View version records. History queries +decode only the selected View's distinct records, preserving history order, +revision filtering, editorial metadata and head guards. Legacy derived records +rebuild under the history lock; Git commit traversal remains linear. Privacy +replacement of the history repository removes these records and unreachable +blobs together, rather than leaving a second history outside the purge boundary. + ### Runnable local CLI Build from `ts` with `pnpm exec fluid-build packages\memory\service -t build` @@ -1109,10 +1217,11 @@ time, and event time. Events can link to durable document sources without copying source content. `searchEvents` lazily reconciles a separate per-corpus KnowPro content index of event projections with the log, including after restart. It uses structured knowledge and message search rather than lexical -event scoring; a missing or outdated index is rebuilt, and model/indexing -failures are reported instead of silently falling back. An outdated generation -is physically removed before a rebuild, including when extraction subsequently -fails, so searches cannot recover or use that generation. Ledger-backed event +event scoring; a missing index is rebuilt and a compatible index appends only +new events. Model/indexing failures are reported instead of silently falling +back. An append failure preserves the previous publication for a later retry, +but does not serve it as a successful up-to-date result. Eligibility changes +physically purge old generations before subsequent search. Ledger-backed event ID tags restrict all requested provenance and date filters before KnowPro ranks and limits results. `authorities` filters by the producer-supplied `metadata.authority` label before ranking, and that label is also indexed as diff --git a/ts/packages/memory/service/src/batchImport.ts b/ts/packages/memory/service/src/batchImport.ts index ac957d1cbe..9de6f43934 100644 --- a/ts/packages/memory/service/src/batchImport.ts +++ b/ts/packages/memory/service/src/batchImport.ts @@ -89,6 +89,7 @@ interface StoredBatch extends MemoryBatchImport { } interface BatchHost { + withIngestionGroup?(operation: () => Promise): Promise; ingestDocument( request: DocumentIngestRequest, signal?: AbortSignal, @@ -329,7 +330,7 @@ export class MemoryBatchStore { string, { controller: AbortController; task: Promise; batch: StoredBatch } >(); - private readonly writes = new Map>(); + private readonly fileAccess = new Map>(); private readonly forgettingSources = new Set(); private readonly maintenanceHolds = new Map(); public constructor( @@ -368,20 +369,31 @@ export class MemoryBatchStore { return path.join(this.root, "batches", `${batchId}.json`); } - private async persist(batch: StoredBatch): Promise { - const previous = this.writes.get(batch.batchId) ?? Promise.resolve(); - const task = previous - .catch(() => undefined) - .then(() => this.write(batch)); - this.writes.set(batch.batchId, task); + private async accessFile( + batchId: string, + operation: () => Promise, + ): Promise { + // Windows reads can prevent atomic replacement; durable reads and + // publication share the same owner-local gate, not an in-memory fallback. + const previous = this.fileAccess.get(batchId) ?? Promise.resolve(); + const task = previous.then(operation); + const settled = task.then( + () => undefined, + () => undefined, + ); + this.fileAccess.set(batchId, settled); try { - await task; + return await task; } finally { - if (this.writes.get(batch.batchId) === task) - this.writes.delete(batch.batchId); + if (this.fileAccess.get(batchId) === settled) + this.fileAccess.delete(batchId); } } + private persist(batch: StoredBatch): Promise { + return this.accessFile(batch.batchId, () => this.write(batch)); + } + private async write(batch: StoredBatch): Promise { const file = this.file(batch.batchId); batch.updatedAt = new Date().toISOString(); @@ -389,7 +401,11 @@ export class MemoryBatchStore { } private async load(batchId: string): Promise { - const batch = decoded(await readFile(this.file(batchId), "utf8")); + const batch = decoded( + await this.accessFile(batchId, () => + readFile(this.file(batchId), "utf8"), + ), + ); for (const member of batch.members) { const index = Number(member.memberId); const document = batch.request.documents[index]; @@ -654,9 +670,7 @@ export class MemoryBatchStore { continue; member.state = "failed"; member.reason = boundedMessage( - error instanceof Error - ? error.message - : "Batch processing failed", + error instanceof Error ? error.message : String(error), ); } await this.persist(batch); @@ -669,20 +683,25 @@ export class MemoryBatchStore { } private async run(batch: StoredBatch, signal: AbortSignal): Promise { - // Two acquired documents at a time; ordinary ingestion keeps corpus commit serialization. + // Keep acquisition concurrency bounded; the host may share index publication. const pending = batch.members.filter( (member) => member.state === "pending", ); for (let index = 0; index < pending.length; index += 2) { - await Promise.all( - pending.slice(index, index + 2).map(async (member) => { - if (signal.aborted || member.state === "cancelled") { - member.state = "cancelled"; - return; - } - await this.ingestMember(batch, member, signal); - }), - ); + const ingest = async () => { + await Promise.all( + pending.slice(index, index + 2).map(async (member) => { + if (signal.aborted || member.state === "cancelled") { + member.state = "cancelled"; + return; + } + await this.ingestMember(batch, member, signal); + }), + ); + }; + if (this.host.withIngestionGroup) + await this.host.withIngestionGroup(ingest); + else await ingest(); await this.persist(batch); } this.linkDuplicates(batch); @@ -801,9 +820,7 @@ export class MemoryBatchStore { } catch (error) { member.state = signal.aborted ? "cancelled" : "failed"; member.reason = boundedMessage( - error instanceof Error - ? error.message - : "Acquired document ingestion failed", + error instanceof Error ? error.message : String(error), ); } } @@ -892,7 +909,9 @@ export class MemoryBatchStore { if (sourceId === undefined) await this.cancel(batch.batchId); else await this.cancelSource(corpusId, sourceId); if (sourceId === undefined) { - await rm(this.file(batch.batchId), { force: true }); + await this.accessFile(batch.batchId, () => + rm(this.file(batch.batchId), { force: true }), + ); } else { const current = await this.load(batch.batchId); const forgotten = current.members.filter( @@ -989,5 +1008,6 @@ export class MemoryBatchStore { await Promise.allSettled( [...this.active.values()].map((active) => active.task), ); + await Promise.all(this.fileAccess.values()); } } diff --git a/ts/packages/memory/service/src/boundedIngestionGroup.ts b/ts/packages/memory/service/src/boundedIngestionGroup.ts new file mode 100644 index 0000000000..e5a6addfeb --- /dev/null +++ b/ts/packages/memory/service/src/boundedIngestionGroup.ts @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AsyncLocalStorage } from "node:async_hooks"; + +interface Admission { + item: T; + resolve: () => void; + reject: (error: unknown) => void; +} + +interface Group { + pending: Admission[]; + closed: boolean; + timer?: ReturnType; +} + +export class BoundedIngestionGroup { + private readonly context = new AsyncLocalStorage>(); + + public constructor( + private readonly dispatch: (items: T[]) => Promise, + ) {} + + public async run(operation: () => Promise): Promise { + const group: Group = { pending: [], closed: false }; + await this.context.run(group, async () => { + try { + await operation(); + } finally { + this.flush(group); + } + }); + } + + public submit(item: T): Promise { + const group = this.context.getStore(); + if (!group || group.closed) return this.dispatch([item]); + return new Promise((resolve, reject) => { + group.pending.push({ item, resolve, reject }); + if (group.pending.length === 2) this.flush(group); + else group.timer = setTimeout(() => this.flush(group), 25); + }); + } + + private flush(group: Group): void { + if (group.closed) return; + group.closed = true; + clearTimeout(group.timer); + if (!group.pending.length) return; + const pending = group.pending.splice(0); + void this.dispatch(pending.map((entry) => entry.item)).then( + () => pending.forEach((entry) => entry.resolve()), + (error: unknown) => pending.forEach((entry) => entry.reject(error)), + ); + } +} diff --git a/ts/packages/memory/service/src/eventSegmentStore.ts b/ts/packages/memory/service/src/eventSegmentStore.ts new file mode 100644 index 0000000000..df3835aaa4 --- /dev/null +++ b/ts/packages/memory/service/src/eventSegmentStore.ts @@ -0,0 +1,418 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { + lstat, + mkdir, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { writeRunbookJson } from "./durableRunbookJson.js"; +import type { MemoryEvent } from "./types.js"; + +interface EventReference { + sequence: number; + identity: string; + sha256: string; + file: string; +} + +interface EventDescriptor { + eventStorageVersion: 1; + sequence: number; + eligibilityRevision: number; + suppressionHash?: string; + pendingPurge?: true; +} + +interface Suppressions { + deleted: { sequence: number; identity: string }[]; + records: T[]; +} + +const digestPattern = /^[a-f0-9]{64}$/; +const eventFilePattern = /^(\d{16})-([a-f0-9]{64})-([a-f0-9]{64})\.jsonl$/; +const hash = (value: string | Buffer) => + createHash("sha256").update(value).digest("hex"); +export type EventStorageFaultPoint = + | "segment-written" + | "descriptor-prepared" + | "purge-started" + | "purge-finished"; + +export class EventSegmentStore { + private descriptor?: EventDescriptor; + private suppression: Suppressions = { + deleted: [], + records: [], + }; + private lookup = new Map(); + private tail = Promise.resolve(); + private readonly directory: string; + + public constructor( + private readonly file: string, + private readonly purgeIndexes: () => Promise, + private readonly checkpoint?: ( + point: EventStorageFaultPoint, + ) => Promise, + ) { + this.directory = path.join(path.dirname(file), "event-segments"); + } + + private async access(operation: () => Promise): Promise { + const result = this.tail.then(operation); + this.tail = result.then( + () => {}, + () => {}, + ); + return result; + } + + private async files(): Promise { + try { + return await readdir(this.directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + } + + private async readBlob(file: string, sha256: string): Promise { + const full = path.join(this.directory, file); + const info = await lstat(full); + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) + throw new Error("Unsafe canonical event segment"); + const bytes = await readFile(full); + if (hash(bytes) !== sha256) + throw new Error("Corrupt canonical event segment"); + return JSON.parse(bytes.toString("utf8")); + } + + private validateDescriptor( + value: unknown, + ): asserts value is EventDescriptor { + if ( + !value || + typeof value !== "object" || + !("eventStorageVersion" in value) || + value.eventStorageVersion !== 1 + ) + throw new Error( + "Unsupported canonical event store; reset/reimport explicitly (no automatic migration)", + ); + if ( + !("sequence" in value) || + !Number.isSafeInteger(value.sequence) || + Number(value.sequence) < 0 || + !("eligibilityRevision" in value) || + !Number.isSafeInteger(value.eligibilityRevision) || + Number(value.eligibilityRevision) < 0 || + ("pendingPurge" in value && value.pendingPurge !== true) || + ("suppressionHash" in value && + (typeof value.suppressionHash !== "string" || + !digestPattern.test(value.suppressionHash))) + ) + throw new Error("Invalid canonical event descriptor"); + } + + private validateSuppressions( + value: unknown, + sequence: number, + ): asserts value is Suppressions { + if ( + !value || + typeof value !== "object" || + !("records" in value) || + !Array.isArray(value.records) || + !("deleted" in value) || + !Array.isArray(value.deleted) + ) + throw new Error("Invalid canonical event suppressions"); + const sequences = new Set(); + const identities = new Set(); + for (const entry of value.deleted) { + if ( + !entry || + typeof entry !== "object" || + !Number.isSafeInteger(entry.sequence) || + entry.sequence < 1 || + entry.sequence > sequence || + sequences.has(entry.sequence) || + typeof entry.identity !== "string" || + !digestPattern.test(entry.identity) || + identities.has(entry.identity) + ) + throw new Error("Invalid canonical event tombstone"); + sequences.add(entry.sequence); + identities.add(entry.identity); + } + } + + private async rebuildLookup(descriptor: EventDescriptor): Promise { + const deleted = new Set( + this.suppression.deleted.map((entry) => entry.sequence), + ); + const sequences = new Set(); + this.lookup.clear(); + for (const file of await this.files()) { + const match = eventFilePattern.exec(file); + if (!match) { + if (!/^suppression-[a-f0-9]{64}\.json$/.test(file)) + throw new Error("Unexpected canonical event segment file"); + continue; + } + const sequence = Number(match[1]); + if (!Number.isSafeInteger(sequence) || sequence < 1) + throw new Error("Invalid event segment sequence"); + if (sequence > descriptor.sequence || deleted.has(sequence)) + continue; + if (sequences.has(sequence) || this.lookup.has(match[2])) + throw new Error("Duplicate canonical event segment"); + sequences.add(sequence); + this.lookup.set(match[2], { + sequence, + identity: match[2], + sha256: match[3], + file, + }); + } + if (sequences.size !== descriptor.sequence - deleted.size) + throw new Error("Missing committed canonical event segment"); + } + + private async load(): Promise { + if (!this.descriptor) { + let value: unknown; + try { + value = JSON.parse(await readFile(this.file, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + if (error instanceof SyntaxError) + throw new Error( + "Incompatible canonical event store; reset/reimport explicitly (no automatic migration)", + ); + throw error; + } + value = { + eventStorageVersion: 1, + sequence: 0, + eligibilityRevision: 0, + }; + } + this.validateDescriptor(value); + if (value.suppressionHash) { + const suppression = await this.readBlob( + `suppression-${value.suppressionHash}.json`, + value.suppressionHash, + ); + this.validateSuppressions(suppression, value.sequence); + this.suppression = suppression; + } + await this.rebuildLookup(value); + this.descriptor = value; + if (!value.pendingPurge) await this.collect(value); + } + if (this.descriptor.pendingPurge) + await this.finishPurge(this.descriptor); + return this.descriptor; + } + + private async publish(value: EventDescriptor): Promise { + await this.checkpoint?.("descriptor-prepared"); + await writeRunbookJson(this.file, `${JSON.stringify(value)}\n`); + this.descriptor = value; + } + + private async collect(descriptor: EventDescriptor): Promise { + const retained = new Set( + [...this.lookup.values()].map((reference) => reference.file), + ); + if (descriptor.suppressionHash) + retained.add(`suppression-${descriptor.suppressionHash}.json`); + for (const file of await this.files()) { + if ( + !eventFilePattern.test(file) && + !/^suppression-[a-f0-9]{64}\.json$/.test(file) + ) + throw new Error("Unexpected canonical event segment file"); + if (!retained.has(file)) await rm(path.join(this.directory, file)); + } + } + + private async finishPurge(value: EventDescriptor): Promise { + await this.checkpoint?.("purge-started"); + await this.purgeIndexes(); + await this.collect(value); + await this.checkpoint?.("purge-finished"); + const ready = { ...value }; + delete ready.pendingPurge; + await this.publish(ready); + } + + private async read(reference: EventReference): Promise { + const value = await this.readBlob(reference.file, reference.sha256); + if ( + !value || + typeof value !== "object" || + !("eventId" in value) || + typeof value.eventId !== "string" || + hash(value.eventId) !== reference.identity + ) + throw new Error("Canonical event identity mismatch"); + return value as MemoryEvent; + } + + public recover(): Promise { + return this.access(async () => { + await this.load(); + }); + } + + public get(eventId: string): Promise { + return this.access(async () => { + await this.load(); + const reference = this.lookup.get(hash(eventId)); + return reference ? this.read(reference) : undefined; + }); + } + + public records(): Promise<{ + events: MemoryEvent[]; + suppressions: TSuppression[]; + }> { + return this.access(async () => { + await this.load(); + const events: MemoryEvent[] = []; + for (const reference of [...this.lookup.values()].sort( + (a, b) => a.sequence - b.sequence, + )) + events.push(await this.read(reference)); + return { + events, + suppressions: structuredClone(this.suppression.records), + }; + }); + } + + public version(): Promise<{ + sequence: number; + eligibilityRevision: number; + }> { + return this.access(async () => { + const value = await this.load(); + return { + sequence: value.sequence, + eligibilityRevision: value.eligibilityRevision, + }; + }); + } + + public suppressions(): Promise { + return this.access(async () => { + await this.load(); + return structuredClone(this.suppression.records); + }); + } + + public sequenceFor(eventId: string): number { + return this.lookup.get(hash(eventId))?.sequence ?? 0; + } + + public append(event: MemoryEvent): Promise { + return this.access(async () => { + const current = await this.load(); + const identity = hash(event.eventId); + if ( + this.lookup.has(identity) || + this.suppression.deleted.some( + (entry) => entry.identity === identity, + ) + ) + throw new Error("Duplicate canonical event identity"); + const bytes = `${JSON.stringify(event)}\n`; + const sha256 = hash(bytes); + const sequence = current.sequence + 1; + if (!Number.isSafeInteger(sequence)) + throw new Error("Canonical event sequence exhausted"); + const file = `${String(sequence).padStart(16, "0")}-${identity}-${sha256}.jsonl`; + await mkdir(this.directory, { recursive: true }); + await writeFile(path.join(this.directory, file), bytes, { + flag: "wx", + }); + let committed = false; + try { + await this.checkpoint?.("segment-written"); + await this.publish({ ...current, sequence }); + committed = true; + this.lookup.set(identity, { sequence, identity, sha256, file }); + } finally { + if (!committed) + await rm(path.join(this.directory, file), { force: true }); + } + }); + } + + public replace( + events: MemoryEvent[], + suppressions: TSuppression[], + ): Promise { + return this.access(async () => { + const current = await this.load(); + const ids = new Set(events.map((event) => hash(event.eventId))); + if ( + ids.size !== events.length || + events.some((event) => !this.lookup.has(hash(event.eventId))) + ) + throw new Error( + "Event compaction may only retain existing immutable events", + ); + const deleted = [...this.lookup.values()].filter( + (reference) => !ids.has(reference.identity), + ); + const suppression: Suppressions = { + deleted: [ + ...this.suppression.deleted, + ...deleted.map(({ sequence, identity }) => ({ + sequence, + identity, + })), + ], + records: structuredClone(suppressions), + }; + const bytes = `${JSON.stringify(suppression)}\n`; + const suppressionHash = hash(bytes); + await mkdir(this.directory, { recursive: true }); + const blob = path.join( + this.directory, + `suppression-${suppressionHash}.json`, + ); + try { + await writeFile(blob, bytes, { flag: "wx" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") + throw error; + await this.readBlob(path.basename(blob), suppressionHash); + } + const next: EventDescriptor = { + ...current, + eligibilityRevision: current.eligibilityRevision + 1, + suppressionHash, + pendingPurge: true, + }; + if (!Number.isSafeInteger(next.eligibilityRevision)) + throw new Error( + "Canonical event eligibility revision exhausted", + ); + await this.publish(next); + this.suppression = suppression; + for (const reference of deleted) + this.lookup.delete(reference.identity); + await this.finishPurge(next); + }); + } +} diff --git a/ts/packages/memory/service/src/fileMemoryService.ts b/ts/packages/memory/service/src/fileMemoryService.ts index fdf0dc9913..90a2be6725 100644 --- a/ts/packages/memory/service/src/fileMemoryService.ts +++ b/ts/packages/memory/service/src/fileMemoryService.ts @@ -6,11 +6,12 @@ import { recordSearchTiming, runWithSearchTiming, timeSearchStage, + collectJsonSegments, + jsonSegmentDirectory, } from "@typeagent/knowpro"; import { createHash, randomUUID } from "node:crypto"; import { access, - appendFile, cp, mkdir, readFile, @@ -35,6 +36,13 @@ import { pruneChangeReceipts, } from "./changeReceipts.js"; import { CorpusAccess } from "./corpusAccess.js"; +import { EventSegmentStore } from "./eventSegmentStore.js"; +import { BoundedIngestionGroup } from "./boundedIngestionGroup.js"; +import { + RevisionTextStore, + type RevisionTextFaultPoint, + type SerializedTextRevision, +} from "./revisionTextStore.js"; import { configuredSourceSynopsisModel, generateSourceSynopsis, @@ -295,6 +303,8 @@ interface StoredRevision extends SourceRevision { } interface EventIndexState { + sequence?: number; + eligibilityRevision?: number; generation: string; watermark: string; } @@ -334,6 +344,17 @@ interface StoredSource extends SourceDocument { revisions: StoredRevision[]; } +interface AcceptedIngestion { + request: DocumentIngestRequest; + content: string; + contentHash: string; + sourceId: string; + revisionId: string; + job: IngestionJobStatus; + signal: AbortSignal; + execute: () => Promise; +} + interface CorpusManifest { corpus: MemoryCorpus; sources: StoredSource[]; @@ -351,6 +372,19 @@ interface CorpusManifest { activeRevisionId: string; revisions: Array<{ revisionId: string; hasAssets: boolean }>; }; + pendingCorpusClear?: Array<{ + sourceId: string; + revisions: Array<{ revisionId: string; hasAssets: boolean }>; + }>; +} + +interface SerializedCorpusManifest extends Omit { + storageVersion: 1; + sources: Array< + Omit & { + revisions: SerializedTextRevision[]; + } + >; } interface CorpusRuntime { @@ -373,6 +407,7 @@ interface PreparedCorpusIndex { } export interface FileMemoryServiceOptions { + revisionTextCheckpoint?: (point: RevisionTextFaultPoint) => Promise; sourceSynopses?: boolean; sourceSynopsisModel?: SourceSynopsisModel; indexedAnswerModel?: ChatModel; @@ -515,10 +550,6 @@ function hashContent(content: string): string { return createHash("sha256").update(content).digest("hex"); } -function eventIndexWatermark(events: readonly MemoryEvent[]): string { - return hashContent(events.map((event) => event.eventId).join("\n")); -} - function contentPipeline( pipeline: SourceRevision["pipeline"], ): IndexedDocument["pipeline"] { @@ -756,8 +787,16 @@ function suppressedTurnKey( ]); } +type EventSuppressionState = Pick< + CorpusRuntime, + | "eventSuppressions" + | "suppressedEventKeys" + | "suppressedConversations" + | "suppressedTurns" +>; + function isSuppressedEvent( - runtime: CorpusRuntime, + runtime: EventSuppressionState, event: MemoryEvent, ): boolean { return ( @@ -777,7 +816,7 @@ function isSuppressedEvent( } function isSuppressedTurn( - runtime: CorpusRuntime, + runtime: Pick, request: MemoryEventAppendRequest, ): boolean { if (request.turnId === undefined) { @@ -889,35 +928,6 @@ async function readJson(filePath: string): Promise { } } -async function readEventRecords( - filePath: string, -): Promise<(MemoryEvent | EventSuppression)[]> { - let content: string; - try { - content = await readFile(filePath, "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return []; - } - throw error; - } - if (content.trim().length === 0) { - return []; - } - return content - .split(/\r?\n/) - .filter((line) => line.length > 0) - .map((line, index) => { - try { - return JSON.parse(line) as MemoryEvent | EventSuppression; - } catch (error) { - throw new Error( - `Invalid event record at line ${index + 1}: ${String(error)}`, - ); - } - }); -} - async function writeJsonAtomic( filePath: string, value: unknown, @@ -979,9 +989,56 @@ export class FileMemoryService private readonly assetStore: RevisionAssetStore; private readonly batchStore: MemoryBatchStore; private readonly corpora = new Map(); + private readonly corpusAccesses = new Map(); + private readonly eventStores = new Map< + string, + EventSegmentStore + >(); + + private corpusAccess(corpusId: string): CorpusAccess { + let access = this.corpusAccesses.get(corpusId); + if (!access) { + access = new CorpusAccess(() => { + const runtime = this.corpora.get(corpusId); + if (runtime) this.assertRemovalComplete(runtime); + }); + this.corpusAccesses.set(corpusId, access); + } + return access; + } + + private eventStore(corpusId: string): EventSegmentStore { + let store = this.eventStores.get(corpusId); + if (!store) { + store = new EventSegmentStore(this.eventsPath(corpusId), () => + this.removeDerivedIndexRoot( + this.eventIndexRoot(corpusId), + "conversation-events", + ), + ); + this.eventStores.set(corpusId, store); + } + return store; + } private readonly corpusLoads = new Map>(); private readonly indexedReads = new Set>(); + private readonly reindexWork = new Set>(); + private readonly documentRepairs = new Map>(); + private readonly detachedRebuilds = new Map< + string, + { + corpusId: string; + sourceIds: Set; + controller: AbortController; + prepared: Promise; + jobIds?: string[]; + } + >(); private readonly ingestionTails = new Map>(); + private readonly ingestionGroups = + new BoundedIngestionGroup((entries) => + this.dispatchIngestionGroup(entries), + ); private readonly preparingIndexes = new Map< string, { corpusId: string; jobId: string; sourceIds: Set } @@ -1007,11 +1064,14 @@ export class FileMemoryService private synopsisModel: SourceSynopsisModel | undefined; private synopsisModelFailure: string | undefined; private readonly synopsisWork = new Map>(); + private readonly textStores = new Map(); + private readonly revisionTextCheckpoint: FileMemoryServiceOptions["revisionTextCheckpoint"]; public constructor( private readonly rootDirectory: string, options: FileMemoryServiceOptions = {}, ) { + this.revisionTextCheckpoint = options.revisionTextCheckpoint; this.viewPublicationCheckpoint = options.viewPublicationCheckpoint; this.viewDrafts = options.viewDrafts !== false; this.wikiModelEndpoint = options.viewModelEndpoint; @@ -1218,13 +1278,32 @@ export class FileMemoryService directory.isDirectory() && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/.test(directory.name) ) { + await this.cleanManifestStaging(directory.name); await this.viewStore.recover(directory.name); await this.cleanArchivedViewIndexes(directory.name); await this.viewStore.recoverViewBuilds(directory.name); - let manifest = await readJson( - this.manifestPath(directory.name), + let manifest: + | Pick< + CorpusManifest, + | "authorizedSourceRemoval" + | "pendingCorpusClear" + | "indexGeneration" + > + | undefined = await this.readSerializedManifest( + directory.name, ); if (manifest) { + if (manifest.pendingCorpusClear) { + const runtime = await this.getCorpusRuntime( + directory.name, + true, + ); + await this.completeCorpusClear( + directory.name, + runtime, + ); + manifest = runtime.manifest; + } if (manifest.authorizedSourceRemoval) { const runtime = await this.getCorpusRuntime( directory.name, @@ -1240,6 +1319,9 @@ export class FileMemoryService const runtime = await this.getCorpusRuntime( directory.name, ); + await this.textStore(directory.name).purge( + runtime.manifest.sources, + ); await this.reconcileTopics(directory.name, runtime); if (!this.informationalWikiEnabled) continue; await this.enqueueWrite(directory.name, () => @@ -1294,11 +1376,12 @@ export class FileMemoryService this.rootWriteTail, ...this.corpusLoads.values(), ...this.indexedReads, + ...this.reindexWork, ]); // Family maintenance may admit a new build while tracked reads drain. const viewClosure = await Promise.allSettled([this.viewBuilds.close()]); await Promise.allSettled( - [...this.corpora.values()].map((runtime) => runtime.access.idle()), + [...this.corpusAccesses.values()].map((access) => access.idle()), ); await this.releaseLock?.(); this.releaseLock = undefined; @@ -1336,7 +1419,7 @@ export class FileMemoryService documentCount: 0, }; const manifest: CorpusManifest = { corpus, sources: [] }; - await writeJsonAtomic(this.manifestPath(corpusId), manifest); + await this.writeManifest(corpusId, manifest); if (this.viewDrafts) await this.viewStore.initializeViewPublicationPolicy(corpusId); const index = this.createIndex(corpusId); @@ -1369,9 +1452,7 @@ export class FileMemoryService if (!entry.isDirectory()) { continue; } - const manifest = await readJson( - this.manifestPath(entry.name), - ); + const manifest = await this.readSerializedManifest(entry.name); if (manifest !== undefined) { corpora.push(manifest.corpus); } @@ -1486,59 +1567,78 @@ export class FileMemoryService }, sources: [], indexGeneration, + pendingCorpusClear: removedSources.map((source) => ({ + sourceId: source.sourceId, + revisions: source.revisions.map((revision) => ({ + revisionId: revision.revisionId, + hasAssets: !!revision.assets?.length, + })), + })), }; - await writeJsonAtomic( - this.manifestPath(corpusId), - candidateManifest, - ); - const suppressions = this.collectEventSuppressions( - runtime.eventSuppressions, - runtime.events, - { - conversationIds: [ - ...new Set( - runtime.events.flatMap((event) => - event.conversationId === undefined - ? [] - : [event.conversationId], - ), - ), - ], - }, - ); - await this.writeEvents(corpusId, [], suppressions); + await this.writeManifest(corpusId, candidateManifest); runtime.manifest = candidateManifest; runtime.index = candidateIndex; - await this.rememberDocumentIndex(corpusId, runtime); - runtime.events = []; - runtime.eventIdempotency.clear(); - this.setEventSuppressions(runtime, suppressions); - for (const source of removedSources) { - for (const revision of source.revisions) { - if (!revision.assets?.length) continue; - await this.assetStore.removeRevision({ - corpusId, - sourceId: source.sourceId, - revisionId: revision.revisionId, - }); - } - } - for (const source of removedSources) { - await this.viewStore.markSourceRevisionStale( + await this.completeCorpusClear(corpusId, runtime); + return clearedCount; + }); + } + + private async completeCorpusClear( + corpusId: string, + runtime: CorpusRuntime, + ): Promise { + const removedSources = runtime.manifest.pendingCorpusClear; + const indexGeneration = runtime.manifest.indexGeneration; + if (!removedSources || !indexGeneration) + throw new Error( + "Corpus clear lacks its durable purge intent or index generation", + ); + const suppressions = this.collectEventSuppressions( + runtime.eventSuppressions, + runtime.events, + { + conversationIds: [ + ...new Set( + runtime.events.flatMap((event) => + event.conversationId === undefined + ? [] + : [event.conversationId], + ), + ), + ], + }, + ); + await this.writeEvents(corpusId, [], suppressions); + await this.rememberDocumentIndex(corpusId, runtime); + runtime.events = []; + runtime.eventIdempotency.clear(); + this.setEventSuppressions(runtime, suppressions); + for (const source of removedSources) { + for (const revision of source.revisions) { + if (!revision.hasAssets) continue; + await this.assetStore.removeRevision({ corpusId, - source.sourceId, - ); + sourceId: source.sourceId, + revisionId: revision.revisionId, + }); } - await this.removeDerivedIndexRoot( - this.eventIndexRoot(corpusId), - "conversation-events", - ); - await this.removeInactiveIndexGenerations( + } + for (const source of removedSources) { + await this.viewStore.markSourceRevisionStale( corpusId, - indexGeneration, + source.sourceId, ); - return clearedCount; - }); + } + await this.removeDerivedIndexRoot( + this.eventIndexRoot(corpusId), + "conversation-events", + ); + await this.removeInactiveIndexGenerations(corpusId, indexGeneration); + await this.textStore(corpusId).purge(runtime.manifest.sources); + const complete = structuredClone(runtime.manifest); + delete complete.pendingCorpusClear; + await this.writeManifest(corpusId, complete); + runtime.manifest = complete; } public async listSources(corpusId: string): Promise { @@ -1953,25 +2053,34 @@ export class FileMemoryService job.jobId, "memory.ingestion", () => - this.enqueueIngestion(request.corpusId, async () => { - const ingest = () => - timeSearchStage("ingestion.total", () => - this.processIngestion( - request, - content, - contentHash, - sourceId, - revisionId, - job, - controller.signal, - ), - ); - await (this.wikiBudget - ? runWithRequestBudget( - new RequestBudget(this.wikiBudget), - ingest, - ) - : ingest()); + this.ingestionGroups.submit({ + request, + content, + contentHash, + sourceId, + revisionId, + job, + signal: controller.signal, + execute: async () => { + const ingest = () => + timeSearchStage("ingestion.total", () => + this.processIngestion( + request, + content, + contentHash, + sourceId, + revisionId, + job, + controller.signal, + ), + ); + await (this.wikiBudget + ? runWithRequestBudget( + new RequestBudget(this.wikiBudget), + ingest, + ) + : ingest()); + }, }), { corpusId: request.corpusId, sourceId, revisionId }, ) @@ -2012,6 +2121,36 @@ export class FileMemoryService }; } + public withIngestionGroup(operation: () => Promise): Promise { + return this.ingestionGroups.run(operation); + } + + private async dispatchIngestionGroup( + entries: AcceptedIngestion[], + ): Promise { + const corpusId = entries[0]!.request.corpusId; + if (entries.some((entry) => entry.request.corpusId !== corpusId)) { + await Promise.all( + entries.map((entry) => + this.enqueueIngestion( + entry.request.corpusId, + entry.execute, + ), + ), + ); + return; + } + await this.enqueueIngestion(corpusId, async () => { + if ( + entries.length === 2 && + !this.wikiBudget && + (await this.processIngestionGroup(corpusId, entries)) + ) + return; + for (const entry of entries) await entry.execute(); + }); + } + public async replaceSource( request: SourceReplaceRequest, signal?: AbortSignal, @@ -2071,10 +2210,7 @@ export class FileMemoryService confirmationToken, expiresAt, }; - await writeJsonAtomic( - this.manifestPath(corpusId), - candidateManifest, - ); + await this.writeManifest(corpusId, candidateManifest); runtime.manifest = candidateManifest; preview = { corpusId, @@ -2141,10 +2277,7 @@ export class FileMemoryService })), }; delete candidateManifest.pendingSourceForget; - await writeJsonAtomic( - this.manifestPath(request.corpusId), - candidateManifest, - ); + await this.writeManifest(request.corpusId, candidateManifest); runtime.manifest = candidateManifest; this.viewBuilds.forget(request.corpusId, request.sourceId); this.controllers @@ -2258,12 +2391,13 @@ export class FileMemoryService await this.viewStore.markSourceRevisionStale(corpusId, sourceId); await this.batchStore.purge(corpusId, sourceId); await this.purgeSourceJobDiagnostics(corpusId, sourceId); + await this.textStore(corpusId).purge(runtime.manifest.sources); for (const key of this.wikiMaintenanceFailures.keys()) if (key.startsWith(`${corpusId}:`)) this.wikiMaintenanceFailures.delete(key); const complete = structuredClone(runtime.manifest); delete complete.authorizedSourceRemoval; - await writeJsonAtomic(this.manifestPath(corpusId), complete); + await this.writeManifest(corpusId, complete); runtime.manifest = complete; this.resumeBatchMaintenance(corpusId); return { @@ -2507,14 +2641,7 @@ export class FileMemoryService ? {} : { metadata: structuredClone(request.metadata) }), }; - await mkdir(path.dirname(this.eventsPath(request.corpusId)), { - recursive: true, - }); - await appendFile( - this.eventsPath(request.corpusId), - `${JSON.stringify(event)}\n`, - "utf8", - ); + await this.eventStore(request.corpusId).append(event); runtime.events.push(event); runtime.eventIdempotency.set(key, event); result = { event: structuredClone(event), replayed: false }; @@ -2529,14 +2656,33 @@ export class FileMemoryService await this.initialize(); validateIdentifier("corpus ID", corpusId); validateIdentifier("event ID", eventId); - const runtime = await this.getCorpusRuntime(corpusId); - return runtime.access.read(async () => { - const event = runtime.events.find( - (item) => item.eventId === eventId, - ); - if (event && isSuppressedEvent(runtime, event)) - throw new Error("Canonical event evidence is suppressed"); - return event === undefined ? undefined : structuredClone(event); + return this.corpusAccess(corpusId).read(async () => { + const manifest = + this.corpora.get(corpusId)?.manifest ?? + (await this.readSerializedManifest(corpusId)); + if (!manifest) throw new Error(`Unknown corpus '${corpusId}'`); + this.assertRemovalComplete({ manifest }); + const store = this.eventStore(corpusId); + const event = await store.get(eventId); + if (!event) return undefined; + this.validateStoredEvent(event, corpusId); + let suppressionState: EventSuppressionState | undefined = + this.corpora.get(corpusId); + if (!suppressionState) { + const suppressions = await store.suppressions(); + for (const record of suppressions) + this.validateStoredEventSuppression(record); + suppressionState = { + eventSuppressions: [], + suppressedEventKeys: new Set(), + suppressedConversations: new Set(), + suppressedTurns: new Set(), + }; + this.setEventSuppressions(suppressionState, suppressions); + } + return isSuppressedEvent(suppressionState, event) + ? undefined + : event; }); } @@ -7006,10 +7152,7 @@ export class FileMemoryService ), ); } - await writeJsonAtomic( - this.manifestPath(request.corpusId), - candidateManifest, - ); + await this.writeManifest(request.corpusId, candidateManifest); runtime.manifest = candidateManifest; await this.invalidateMaintainedViews(request.corpusId); }); @@ -7019,40 +7162,148 @@ export class FileMemoryService ); } - private async reindex( + private reindex( + corpusId: string, + sourceId: string | undefined, + signal: AbortSignal, + ): Promise { + const work = this.prepareReindex(corpusId, sourceId, signal); + this.reindexWork.add(work); + void work.then( + () => this.reindexWork.delete(work), + () => this.reindexWork.delete(work), + ); + return work; + } + + private async prepareReindex( corpusId: string, sourceId: string | undefined, signal: AbortSignal, ): Promise { await this.initialize(); validateIdentifier("corpus ID", corpusId); - let result: ReindexResult | undefined; - await this.enqueueWrite(corpusId, async () => { - const runtime = await this.getCorpusRuntime(corpusId); - if ( - sourceId !== undefined && - !runtime.manifest.sources.some( - (source) => source.sourceId === sourceId, + const generation = randomUUID(); + const directory = this.indexDirectory(corpusId, generation); + const controller = new AbortController(); + const bounded = AbortSignal.any([signal, controller.signal]); + let finishPreparation!: () => void; + const prepared = new Promise((resolve) => { + finishPreparation = resolve; + }); + let published = false; + try { + const capture = await this.enqueueWrite(corpusId, async () => { + if (this.closed) throw new Error("Memory service is closed"); + this.throwIfAborted(bounded); + const runtime = await this.getCorpusRuntime(corpusId); + if ( + sourceId !== undefined && + !runtime.manifest.sources.some( + (source) => source.sourceId === sourceId, + ) ) - ) { - throw new Error(`Unknown source '${sourceId}'`); + throw new Error(`Unknown source '${sourceId}'`); + if (runtime.manifest.indexGeneration !== undefined) + await classifyIndexSchema( + this.indexDirectory( + corpusId, + runtime.manifest.indexGeneration, + ), + "documents", + false, + ); + const manifest = structuredClone(runtime.manifest); + const sourceIds = new Set( + manifest.sources.map((source) => source.sourceId), + ); + this.controllers.set(generation, controller); + this.detachedRebuilds.set(directory, { + corpusId, + sourceIds, + controller, + prepared, + }); + this.preparingIndexes.set(directory, { + corpusId, + sourceIds, + jobId: generation, + }); + return { + runtime, + documents: this.activeDocuments(manifest), + baseGeneration: manifest.indexGeneration, + baseVersion: this.indexVersion(manifest), + }; + }); + const index = this.indexFactory(corpusId, directory); + try { + await mkdir(directory, { recursive: true }); + const rebuild = () => + index.rebuild(capture.documents, bounded, async () => {}); + await timeSearchStage("reindex.prepare", () => + this.wikiBudget + ? runWithRequestBudget( + new RequestBudget(this.wikiBudget), + rebuild, + ) + : rebuild(), + ); + this.throwIfAborted(bounded); + await stampIndexSchema( + directory, + "documents", + capture.documents.length > 0, + ); + } finally { + finishPreparation(); } - const candidateManifest = structuredClone(runtime.manifest); - delete candidateManifest.pendingSourceForget; - await this.rebuildAndActivate( - corpusId, - runtime, - candidateManifest, - signal, + return await this.enqueueWrite(corpusId, () => + timeSearchStage("reindex.commit", async () => { + this.throwIfAborted(bounded); + const runtime = capture.runtime; + if ( + runtime.manifest.indexGeneration !== + capture.baseGeneration || + this.indexVersion(runtime.manifest) !== + capture.baseVersion + ) + throw new ViewBuildStaleError( + "Corpus source index changed during reindex preparation", + ); + const manifest = structuredClone(runtime.manifest); + manifest.indexGeneration = generation; + manifest.corpus = { + ...manifest.corpus, + updatedAt: now(), + status: "ready", + documentCount: manifest.sources.length, + }; + await this.writeManifest(corpusId, manifest); + published = true; + runtime.manifest = manifest; + runtime.index = index; + await this.rememberDocumentIndex(corpusId, runtime); + await this.removeInactiveIndexGenerations( + corpusId, + generation, + ); + return { + corpusId, + ...(sourceId === undefined ? {} : { sourceId }), + sourceCount: manifest.sources.length, + indexVersion: this.indexVersion(manifest), + }; + }), ); - result = { - corpusId, - ...(sourceId === undefined ? {} : { sourceId }), - sourceCount: runtime.manifest.sources.length, - indexVersion: this.indexVersion(runtime.manifest), - }; - }); - return result!; + } finally { + finishPreparation(); + if (!published) + await rm(directory, { recursive: true, force: true }); + this.preparingIndexes.delete(directory); + this.detachedRebuilds.delete(directory); + this.controllers.delete(generation); + } } private async rebuildAndActivate( @@ -7104,10 +7355,7 @@ export class FileMemoryService status: "ready", documentCount: candidateManifest.sources.length, }; - await writeJsonAtomic( - this.manifestPath(corpusId), - candidateManifest, - ); + await this.writeManifest(corpusId, candidateManifest); runtime.manifest = candidateManifest; runtime.index = candidateIndex; await this.rememberDocumentIndex(corpusId, runtime); @@ -7126,6 +7374,34 @@ export class FileMemoryService const root = this.indexDirectory(corpusId); const removal = this.corpora.get(corpusId)?.manifest.authorizedSourceRemoval; + const sources = new Set( + this.corpora + .get(corpusId) + ?.manifest.sources.map((source) => source.sourceId), + ); + for (const [directory, preparation] of this.detachedRebuilds) { + if ( + preparation.corpusId !== corpusId || + directory === this.indexDirectory(corpusId, activeGeneration) || + !(removal + ? preparation.sourceIds.has(removal.sourceId) + : [...preparation.sourceIds].some((id) => !sources.has(id))) + ) + continue; + preparation.controller.abort( + new ViewBuildStaleError( + "Source removal invalidated reindex preparation", + ), + ); + if (this.corpora.get(corpusId)?.manifest.pendingCorpusClear) + for (const jobId of preparation.jobIds ?? []) + this.controllers + .get(jobId) + ?.abort(new Error("Corpus cleared during ingestion")); + // Wait for provider writes, not publication (which needs this reservation). + await preparation.prepared; + this.preparingIndexes.delete(directory); + } // Detached generations still contain retained sources and must be // cancelled and purged when any of that evidence is forgotten. if (removal) @@ -7156,6 +7432,7 @@ export class FileMemoryService .filter( (entry) => entry.isDirectory() && + entry.name !== jsonSegmentDirectory && entry.name !== activeGeneration && !this.preparingIndexes.has(path.join(root, entry.name)), ) @@ -7168,6 +7445,18 @@ export class FileMemoryService }); }), ); + if ( + ![ + ...this.preparingIndexes.keys(), + ...this.detachedRebuilds.keys(), + ].some( + (directory) => + path.dirname(directory) === root && + directory !== + this.indexDirectory(corpusId, activeGeneration), + ) + ) + await collectJsonSegments(root); } private indexVersion(manifest: CorpusManifest): string { @@ -7245,14 +7534,16 @@ export class FileMemoryService if (!entry.isDirectory()) { continue; } - const manifestPath = this.manifestPath(entry.name); - const manifest = await readJson(manifestPath); + const manifest = await this.readSerializedManifest(entry.name); if (manifest?.changes === undefined) { continue; } const changes = pruneChangeReceipts(manifest.changes); if (changes.length !== manifest.changes.length) { - await writeJsonAtomic(manifestPath, { ...manifest, changes }); + await writeRunbookJson( + this.manifestPath(entry.name), + `${JSON.stringify({ ...manifest, changes }, undefined, 2)}\n`, + ); } } } @@ -7276,6 +7567,256 @@ export class FileMemoryService ); } + private async captureIngestionGroup( + corpusId: string, + entries: AcceptedIngestion[], + generation: string, + controller: AbortController, + prepared: Promise, + ) { + return this.enqueueWrite(corpusId, async () => { + const runtime = await this.getCorpusRuntime(corpusId); + const ids = new Set(entries.map((entry) => entry.sourceId)); + if ( + ids.size !== entries.length || + entries.some( + (entry) => + entry.signal.aborted || + entry.request.source.assets?.length || + entry.request.pipeline?.expectedActiveRevisionId !== + undefined || + runtime.manifest.sources.some( + (source) => source.sourceId === entry.sourceId, + ), + ) + ) + return undefined; + const directory = this.indexDirectory(corpusId, generation); + const index = this.indexFactory(corpusId, directory); + const existingGeneration = runtime.manifest.indexGeneration; + const append = runtime.manifest.sources.length > 0; + if ( + append && + (!existingGeneration || + !index.append || + (await classifyIndexSchema( + this.indexDirectory(corpusId, existingGeneration), + "documents", + true, + )) !== "current") + ) + return undefined; + const additions = entries.map((entry) => { + this.assertSourceIngestionAllowed(runtime, entry.sourceId); + const { source, revision } = createStoredSource( + entry.request, + entry.content, + entry.contentHash, + entry.sourceId, + entry.revisionId, + this.mimeType(entry.request.source.sourceType), + undefined, + ); + revision.assets = []; + return { + source, + revision, + content: entry.content, + pipeline: contentPipeline(revision.pipeline), + }; + }); + const candidateManifest = structuredClone(runtime.manifest); + candidateManifest.sources.push( + ...additions.map((document) => document.source), + ); + const sourceIds = new Set( + candidateManifest.sources.map((source) => source.sourceId), + ); + await mkdir(directory, { recursive: true }); + this.preparingIndexes.set(directory, { + corpusId, + jobId: entries[0]!.job.jobId, + sourceIds, + }); + this.detachedRebuilds.set(directory, { + corpusId, + sourceIds, + controller, + prepared, + jobIds: entries.map((entry) => entry.job.jobId), + }); + if (append) + await cp( + this.indexDirectory(corpusId, existingGeneration!), + directory, + { recursive: true }, + ); + for (const entry of entries) + await this.updateJob(entry.job, "building-indexes", { + completed: 0, + total: 1, + operation: append ? "append" : "rebuild", + message: "Building shared two-source index", + }); + return { + runtime, + index, + additions, + append, + documents: this.activeDocuments(candidateManifest), + baseGeneration: existingGeneration, + baseVersion: this.indexVersion(runtime.manifest), + }; + }); + } + + private async processIngestionGroup( + corpusId: string, + entries: AcceptedIngestion[], + ): Promise { + const generation = randomUUID(); + const directory = this.indexDirectory(corpusId, generation); + const controller = new AbortController(); + const signal = AbortSignal.any([ + controller.signal, + ...entries.map((entry) => entry.signal), + ]); + let finishPreparation!: () => void; + const prepared = new Promise((resolve) => { + finishPreparation = resolve; + }); + let committed = false; + try { + const capture = await this.captureIngestionGroup( + corpusId, + entries, + generation, + controller, + prepared, + ); + if (!capture) return false; + const progress = async (value: JobProgress) => { + for (const entry of entries) { + if (signal.aborted) return; + await this.updateJob( + entry.job, + value.stage ?? "building-indexes", + { + ...value, + message: `Shared two-source index: ${value.message ?? "preparing"}`, + }, + ); + } + }; + try { + signal.throwIfAborted(); + // Drain the actual provider, even if it ignores abort. Privacy + // cleanup waits for this promise, never for publication. + await timeSearchStage("ingestion.prepare.group", () => + capture.append + ? capture.index.append!( + capture.additions, + signal, + progress, + ) + : capture.index.rebuild( + capture.documents, + signal, + progress, + ), + ); + signal.throwIfAborted(); + await stampIndexSchema(directory, "documents", true); + } finally { + finishPreparation(); + } + await this.enqueueWrite(corpusId, async () => { + signal.throwIfAborted(); + if ( + capture.runtime.manifest.indexGeneration !== + capture.baseGeneration || + this.indexVersion(capture.runtime.manifest) !== + capture.baseVersion + ) + throw new ViewBuildStaleError( + "Corpus changed during grouped ingestion preparation", + ); + const manifest = structuredClone(capture.runtime.manifest); + delete manifest.pendingSourceForget; + for (const addition of capture.additions) { + addition.revision.state = "ready"; + addition.revision.indexedAt = now(); + manifest.sources.push(addition.source); + } + manifest.indexGeneration = generation; + manifest.corpus.status = "ready"; + manifest.corpus.updatedAt = now(); + manifest.corpus.documentCount = manifest.sources.length; + manifest.changes = pruneChangeReceipts(manifest.changes); + for (const entry of entries) + await this.updateJob(entry.job, "persisting", { + completed: 1, + total: 1, + message: "Persisting shared corpus metadata", + }); + await this.writeManifest(corpusId, manifest); + capture.runtime.manifest = manifest; + capture.runtime.index = capture.index; + committed = true; + await this.rememberDocumentIndex(corpusId, capture.runtime); + for (const entry of entries) + await this.viewStore.markSourceRevisionStale( + corpusId, + entry.sourceId, + entry.revisionId, + ); + await this.invalidateMaintainedViews(corpusId); + await this.textStore(corpusId).purge(manifest.sources); + await this.removeInactiveIndexGenerations(corpusId, generation); + for (const entry of entries) + await this.updateJob(entry.job, "complete", { + completed: 1, + total: 1, + message: "Ingestion complete", + }); + }); + return true; + } catch (error) { + finishPreparation(); + if ( + !committed && + (signal.aborted || error instanceof ViewBuildStaleError) + ) + return false; + await this.enqueueWrite( + corpusId, + async () => { + for (const entry of entries) + await this.handleIngestionFailure( + entry.job, + entry.signal, + error, + committed, + undefined, + false, + ); + }, + true, + ); + return true; + } finally { + finishPreparation(); + this.detachedRebuilds.delete(directory); + this.preparingIndexes.delete(directory); + if (!committed) + await this.enqueueWrite( + corpusId, + () => rm(directory, { recursive: true, force: true }), + true, + ); + } + } + private async processIngestion( request: DocumentIngestRequest, content: string, @@ -7285,6 +7826,12 @@ export class FileMemoryService job: IngestionJobStatus, signal: AbortSignal, ): Promise { + const preparationController = new AbortController(); + signal = AbortSignal.any([signal, preparationController.signal]); + let finishPreparation!: () => void; + const prepared = new Promise((resolve) => { + finishPreparation = resolve; + }); const indexGeneration = randomUUID(); const candidateDirectory = this.indexDirectory( request.corpusId, @@ -7378,6 +7925,17 @@ export class FileMemoryService ), }); await mkdir(candidateDirectory, { recursive: true }); + this.detachedRebuilds.set(candidateDirectory, { + corpusId: request.corpusId, + sourceIds: new Set( + candidateManifest.sources.map( + (item) => item.sourceId, + ), + ), + controller: preparationController, + prepared, + jobIds: [job.jobId], + }); const candidateIndex = this.indexFactory( request.corpusId, candidateDirectory, @@ -7388,7 +7946,10 @@ export class FileMemoryService }); const documents = this.activeDocuments(candidateManifest); const canAppend = - existing === undefined && + (existing === undefined || + (policy === "retainRevisionHistory" && + !request.source.assets?.length && + candidateIndex.replace !== undefined)) && runtime.manifest.sources.length > 0 && runtime.manifest.indexGeneration !== undefined && candidateIndex.append !== undefined && @@ -7462,35 +8023,32 @@ export class FileMemoryService await timeSearchStage("ingestion.prepare", async () => { if (canAppend) { this.throwIfAborted(signal); - await raceWithAbort( - timeSearchStage("ingestion.index.append", () => - candidateIndex.append!( - [ - { - source, - revision, - content, - pipeline: contentPipeline( - revision.pipeline, - ), - }, - ], - signal, - reportProgress, - ), - ), - signal, + const addition = { + source, + revision, + content, + pipeline: contentPipeline(revision.pipeline), + }; + await timeSearchStage("ingestion.index.append", () => + existing && candidateIndex.replace + ? candidateIndex.replace( + addition, + signal, + reportProgress, + ) + : candidateIndex.append!( + [addition], + signal, + reportProgress, + ), ); } else { - await raceWithAbort( - timeSearchStage("ingestion.index.rebuild", () => - candidateIndex.rebuild( - documents, - signal, - reportProgress, - ), + await timeSearchStage("ingestion.index.rebuild", () => + candidateIndex.rebuild( + documents, + signal, + reportProgress, ), - signal, ); } this.throwIfAborted(signal); @@ -7499,7 +8057,9 @@ export class FileMemoryService "documents", candidateManifest.sources.length > 0, ); + finishPreparation(); }); + await collectJsonSegments(path.dirname(candidateDirectory)); await this.enqueueWrite(request.corpusId, () => timeSearchStage("ingestion.commit", async () => { this.throwIfAborted(signal); @@ -7560,8 +8120,8 @@ export class FileMemoryService total: 1, message: "Persisting corpus metadata", }); - await writeJsonAtomic( - this.manifestPath(request.corpusId), + await this.writeManifest( + request.corpusId, committedManifest, ); runtime.manifest = committedManifest; @@ -7580,6 +8140,9 @@ export class FileMemoryService source, existing, ); + await this.textStore(request.corpusId).purge( + runtime.manifest.sources, + ); await this.removeInactiveIndexGenerations( request.corpusId, indexGeneration, @@ -7592,6 +8155,7 @@ export class FileMemoryService }), ); } catch (error) { + finishPreparation(); await this.enqueueWrite( request.corpusId, () => @@ -7606,6 +8170,8 @@ export class FileMemoryService true, ); } finally { + finishPreparation(); + this.detachedRebuilds.delete(candidateDirectory); this.preparingIndexes.delete(candidateDirectory); this.controllers.delete(job.jobId); } @@ -7748,7 +8314,7 @@ export class FileMemoryService const changes = pruneChangeReceipts(runtime.manifest.changes); if (changes.length !== (runtime.manifest.changes ?? []).length) { const candidate = { ...runtime.manifest, changes }; - await writeJsonAtomic(this.manifestPath(corpusId), candidate); + await this.writeManifest(corpusId, candidate); runtime.manifest = candidate; } return operation(); @@ -7761,12 +8327,14 @@ export class FileMemoryService ready: (runtime: CorpusRuntime) => Promise, prepare: (runtime: CorpusRuntime) => Promise, operation: (runtime: CorpusRuntime) => Promise, + detachedPreparation = false, ): Promise { const read = this.readPreparedIndex( corpusId, ready, prepare, operation, + detachedPreparation, ); this.indexedReads.add(read); void read.then( @@ -7781,9 +8349,10 @@ export class FileMemoryService ready: (runtime: CorpusRuntime) => Promise, prepare: (runtime: CorpusRuntime) => Promise, operation: (runtime: CorpusRuntime) => Promise, + detachedPreparation: boolean, ): Promise { const runtime = await this.getCorpusRuntime(corpusId); - while (true) { + for (let attempts = 0; ; attempts++) { const queuedAt = currentSearchTraceId() ? performance.now() : undefined; @@ -7805,7 +8374,13 @@ export class FileMemoryService }; }); if (result.ready) return result.value; - await this.enqueueWrite(corpusId, () => prepare(runtime)); + if (detachedPreparation) { + if (attempts >= 3) + throw new ViewBuildStaleError( + "Corpus index repeatedly changed during read preparation", + ); + await prepare(runtime); + } else await this.enqueueWrite(corpusId, () => prepare(runtime)); } } @@ -7816,12 +8391,49 @@ export class FileMemoryService return this.enqueueRead( corpusId, (runtime) => this.documentIndexReady(corpusId, runtime), - (runtime) => this.ensureDocumentIndex(corpusId, runtime), + (runtime) => this.prepareDocumentReadIndex(corpusId, runtime), operation, + true, ); } - private assertRemovalComplete(runtime: CorpusRuntime): void { + private async prepareDocumentReadIndex( + corpusId: string, + runtime: CorpusRuntime, + ): Promise { + let repair = this.documentRepairs.get(corpusId); + if (!repair) { + repair = (async () => { + const ready = await this.enqueueWrite(corpusId, () => + this.ensureDocumentIndex(corpusId, runtime, false), + ); + if (!ready) + await this.reindex( + corpusId, + undefined, + new AbortController().signal, + ); + })(); + this.documentRepairs.set(corpusId, repair); + } + try { + await repair; + } finally { + if (this.documentRepairs.get(corpusId) === repair) + this.documentRepairs.delete(corpusId); + } + } + + private assertRemovalComplete(runtime: { + manifest: Pick< + CorpusManifest, + "pendingCorpusClear" | "authorizedSourceRemoval" + >; + }): void { + if (runtime.manifest.pendingCorpusClear) + throw new ViewBuildStaleError( + "Corpus clear is pending purge recovery; corpus reads and writes are withheld", + ); if (runtime.manifest.authorizedSourceRemoval) throw new ViewBuildStaleError( "Authorized source removal is pending purge recovery; corpus reads and writes are withheld", @@ -7834,6 +8446,7 @@ export class FileMemoryService ): Promise { const cached = this.corpora.get(corpusId); if (cached !== undefined) { + await this.eventStore(corpusId).recover(); if (!recoveringRemoval) this.assertRemovalComplete(cached); return cached; } @@ -7853,13 +8466,12 @@ export class FileMemoryService } private async loadCorpusRuntime(corpusId: string): Promise { - const manifest = await readJson( - this.manifestPath(corpusId), - ); + const manifest = await this.readManifest(corpusId); if (manifest === undefined) { throw new Error(`Unknown corpus '${corpusId}'`); } - const records = await readEventRecords(this.eventsPath(corpusId)); + const persisted = await this.eventStore(corpusId).records(); + const records = [...persisted.events, ...persisted.suppressions]; const events: MemoryEvent[] = []; const eventSuppressions: EventSuppression[] = []; const eventIdempotency = new Map(); @@ -7896,7 +8508,7 @@ export class FileMemoryService suppressedEventKeys: new Set(), suppressedConversations: new Set(), suppressedTurns: new Set(), - access: new CorpusAccess(() => this.assertRemovalComplete(runtime)), + access: this.corpusAccess(corpusId), }; this.setEventSuppressions(runtime, eventSuppressions); for (const event of events) { @@ -7923,8 +8535,9 @@ export class FileMemoryService private async ensureDocumentIndex( corpusId: string, runtime: CorpusRuntime, - ): Promise { - if (await this.documentIndexReady(corpusId, runtime)) return; + allowRebuild = true, + ): Promise { + if (await this.documentIndexReady(corpusId, runtime)) return true; const generation = runtime.manifest.indexGeneration; if (generation !== undefined) { validateIdentifier("index generation", generation); @@ -7950,8 +8563,9 @@ export class FileMemoryService }, ); await this.rememberDocumentIndex(corpusId, runtime); - return; + return true; } + if (!allowRebuild) return false; await rm(directory, { recursive: true, force: true }); } else if (runtime.manifest.sources.length === 0) { await timeSearchStage( @@ -7963,8 +8577,9 @@ export class FileMemoryService }, ); await this.rememberDocumentIndex(corpusId, runtime); - return; + return true; } + if (!allowRebuild) return false; await timeSearchStage( "index.rebuild", () => @@ -7976,6 +8591,7 @@ export class FileMemoryService ), { corpusId, indexKind: "documents" }, ); + return true; } private async documentIndexReady( @@ -8076,7 +8692,7 @@ export class FileMemoryService throw error; } for (const entry of entries) { - if (entry.isDirectory()) { + if (entry.isDirectory() && entry.name !== jsonSegmentDirectory) { await classifyIndexSchema( path.join(root, entry.name), indexKind, @@ -8091,7 +8707,8 @@ export class FileMemoryService corpusId: string, runtime: CorpusRuntime, ): Promise { - const watermark = eventIndexWatermark(runtime.events); + const version = await this.eventStore(corpusId).version(); + const watermark = `${version.sequence}:${version.eligibilityRevision}`; const root = this.eventIndexRoot(corpusId); const state = await readJson( path.join(root, "state.json"), @@ -8110,6 +8727,12 @@ export class FileMemoryService } } if (state?.watermark !== watermark) { + if ( + state?.eligibilityRevision === version.eligibilityRevision && + state.sequence !== undefined && + state.sequence <= version.sequence + ) + return state; await this.removeDerivedIndexRoot(root, "conversation-events"); return undefined; } @@ -8121,8 +8744,10 @@ export class FileMemoryService runtime: CorpusRuntime, ): Promise { const root = this.eventIndexRoot(corpusId); - const state = await this.purgeObsoleteEventIndex(corpusId, runtime); - if (state !== undefined) { + const store = this.eventStore(corpusId); + const version = await store.version(); + let state = await this.purgeObsoleteEventIndex(corpusId, runtime); + if (state !== undefined && state.sequence === version.sequence) { try { await access( path.join( @@ -8142,6 +8767,7 @@ export class FileMemoryService } } await this.removeDerivedIndexRoot(root, "conversation-events"); + state = undefined; } await mkdir(root, { recursive: true }); const generation = randomUUID(); @@ -8149,11 +8775,34 @@ export class FileMemoryService await mkdir(directory); try { const index = this.eventIndexFactory(corpusId, directory); - await index.rebuild( - runtime.events.map((event) => this.eventDocument(event)), - new AbortController().signal, - async () => {}, - ); + if ( + state && + index.append && + state.eligibilityRevision === version.eligibilityRevision + ) { + await cp( + this.eventIndexDirectory(corpusId, state.generation), + directory, + { recursive: true }, + ); + await index.append( + runtime.events + .filter( + (event) => + store.sequenceFor(event.eventId) > + (state.sequence ?? 0), + ) + .map((event) => this.eventDocument(event)), + new AbortController().signal, + async () => {}, + ); + } else { + await index.rebuild( + runtime.events.map((event) => this.eventDocument(event)), + new AbortController().signal, + async () => {}, + ); + } await stampIndexSchema( directory, "conversation-events", @@ -8162,13 +8811,16 @@ export class FileMemoryService await writeFile(path.join(directory, "ready"), ""); await writeJsonAtomic(path.join(root, "state.json"), { generation, - watermark: eventIndexWatermark(runtime.events), + watermark: `${version.sequence}:${version.eligibilityRevision}`, + ...version, } satisfies EventIndexState); const obsolete = ( await readdir(root, { withFileTypes: true }) ).filter( (entry) => - entry.name !== generation && entry.name !== "state.json", + entry.name !== generation && + entry.name !== "state.json" && + entry.name !== jsonSegmentDirectory, ); for (const entry of obsolete) { if (entry.isDirectory()) { @@ -8185,6 +8837,7 @@ export class FileMemoryService force: true, }); } + await collectJsonSegments(root); return index; } catch (error) { await rm(directory, { recursive: true, force: true }); @@ -8476,10 +9129,7 @@ export class FileMemoryService status: "failed", reason: "Interrupted synopsis exhausted its automatic retry; waiting for changed input", }; - await writeJsonAtomic( - this.manifestPath(corpusId), - candidate, - ); + await this.writeManifest(corpusId, candidate); current.manifest = candidate; return undefined; } @@ -8499,10 +9149,7 @@ export class FileMemoryService ? prior.attempts + 1 : 1, }; - await writeJsonAtomic( - this.manifestPath(corpusId), - candidate, - ); + await this.writeManifest(corpusId, candidate); current.manifest = candidate; return { source: structuredClone(source), @@ -8607,10 +9254,7 @@ export class FileMemoryService "Generation failed; scan counts describe submitted input, not accepted semantic coverage.", ], }; - await writeJsonAtomic( - this.manifestPath(corpusId), - candidate, - ); + await this.writeManifest(corpusId, candidate); current.manifest = candidate; }, true, @@ -8675,7 +9319,7 @@ export class FileMemoryService } private setEventSuppressions( - runtime: CorpusRuntime, + runtime: EventSuppressionState, suppressions: EventSuppression[], ): void { runtime.eventSuppressions = suppressions; @@ -8950,41 +9594,10 @@ export class FileMemoryService events: MemoryEvent[], suppressions: EventSuppression[], ): Promise { - await mkdir(path.dirname(this.eventsPath(corpusId)), { - recursive: true, - }); - const records: (MemoryEvent | EventSuppression)[] = [ - ...events, - ...suppressions, - ]; - const value = - records.length === 0 - ? "" - : `${records.map((record) => JSON.stringify(record)).join("\n")}\n`; - const filePath = this.eventsPath(corpusId); - const temporaryPath = `${filePath}.${randomUUID()}.tmp`; - const backupPath = `${filePath}.${randomUUID()}.bak`; - await writeFile(temporaryPath, value, "utf8"); - let hasBackup = false; - try { - await rename(filePath, backupPath); - hasBackup = true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - await rm(temporaryPath, { force: true }); - throw error; - } - } try { - await rename(temporaryPath, filePath); - if (hasBackup) { - await rm(backupPath, { force: true }); - } + await this.eventStore(corpusId).replace(events, suppressions); } catch (error) { - await rm(temporaryPath, { force: true }); - if (hasBackup) { - await rename(backupPath, filePath); - } + this.corpora.delete(corpusId); throw error; } } @@ -9063,6 +9676,91 @@ export class FileMemoryService }); } + private textStore(corpusId: string): RevisionTextStore { + let store = this.textStores.get(corpusId); + if (!store) { + store = new RevisionTextStore( + path.join(this.rootDirectory, corpusId, "revision-text"), + this.revisionTextCheckpoint, + ); + this.textStores.set(corpusId, store); + } + return store; + } + + private async cleanManifestStaging(corpusId: string): Promise { + const directory = path.join(this.rootDirectory, corpusId); + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (!/^manifest\.json\.[a-f0-9-]{36}\.staging$/.test(entry.name)) + continue; + if (!entry.isFile()) + throw new Error("Unsafe staged corpus manifest"); + await rm(path.join(directory, entry.name)); + } + } + + private async readSerializedManifest( + corpusId: string, + ): Promise { + const stored = await readJson( + this.manifestPath(corpusId), + ); + if (!stored) return undefined; + if (stored.storageVersion !== 1) + throw new Error( + `Unsupported corpus manifest storage version '${stored.storageVersion}' for '${corpusId}'; reset/reimport this corpus using a compatible build. Canonical data was not deleted.`, + ); + return stored; + } + + private async readManifest( + corpusId: string, + ): Promise { + const stored = await this.readSerializedManifest(corpusId); + if (!stored) return undefined; + const { storageVersion: _version, ...manifest } = stored; + const store = this.textStore(corpusId); + const sources: StoredSource[] = []; + for (const source of stored.sources) { + const revisions: StoredRevision[] = []; + for (const revision of source.revisions) { + const { text: _text, ...metadata } = revision; + revisions.push({ + ...metadata, + content: await store.read(revision), + }); + } + sources.push({ ...source, revisions }); + } + return { ...manifest, sources }; + } + + private async writeManifest( + corpusId: string, + manifest: CorpusManifest, + ): Promise { + const store = this.textStore(corpusId); + const sources = []; + // Publication is serialized by the corpus reservation. Stage new blobs + // first; a crash can leave only unreferenced blobs, never dangling refs. + for (const source of manifest.sources) { + const revisions = []; + for (const revision of source.revisions) + revisions.push(await store.prepare(revision)); + sources.push({ ...source, revisions }); + } + const stored: SerializedCorpusManifest = { + ...manifest, + storageVersion: 1, + sources, + }; + await store.prepared(); + await writeRunbookJson( + this.manifestPath(corpusId), + `${JSON.stringify(stored, undefined, 2)}\n`, + ); + } + private manifestPath(corpusId: string): string { return path.join(this.rootDirectory, corpusId, manifestFileName); } diff --git a/ts/packages/memory/service/src/indexSchema.ts b/ts/packages/memory/service/src/indexSchema.ts index 11f5959594..efe78cceca 100644 --- a/ts/packages/memory/service/src/indexSchema.ts +++ b/ts/packages/memory/service/src/indexSchema.ts @@ -3,6 +3,7 @@ import { readFile, stat, writeFile } from "node:fs/promises"; import path from "node:path"; +import { isSegmentedJson } from "@typeagent/knowpro"; export type IndexKind = "documents" | "conversation-events" | "views"; @@ -22,6 +23,7 @@ export async function indexFingerprint( const files = [ schemaFileName, semanticFileName, + "eligibility.json", "corpus_embeddings.bin", ...(indexKind === "documents" ? ["document-projection.json"] @@ -69,6 +71,14 @@ async function hasSemanticData(directory: string): Promise { `Malformed semantic index '${filePath}': ${String(error)}`, ); } + if (isSegmentedJson(value)) { + if ( + !value.metadata.messages || + value.metadata.semanticRefs === undefined + ) + throw new Error(`Invalid semantic index '${filePath}'`); + return true; + } if ( typeof value !== "object" || value === null || diff --git a/ts/packages/memory/service/src/knowProCorpusIndex.ts b/ts/packages/memory/service/src/knowProCorpusIndex.ts index 71c9f3858a..e1211f885b 100644 --- a/ts/packages/memory/service/src/knowProCorpusIndex.ts +++ b/ts/packages/memory/service/src/knowProCorpusIndex.ts @@ -16,8 +16,9 @@ import { isInitiativeSubject } from "./initiativeDiscovery.js"; import { indexedTopicOccurrence } from "./topicIncidence.js"; import type { TopicOccurrence } from "./topicTypes.js"; import { symbolicIdentity, usefulWikiSubject } from "./wikiGrouping.js"; -import { access } from "node:fs/promises"; +import { access, readFile, rm } from "node:fs/promises"; import path from "node:path"; +import { writeRunbookJson } from "./durableRunbookJson.js"; import { indexedTextHash, parseIndexedLocator, @@ -135,6 +136,8 @@ function collectPostingScores( terms: string[], requested: Set, typedQuery: boolean, + eligible: (ordinal: number) => boolean, + messageCount: number, ): { scores: Map; typedSubjects: Map> } { const matches = new Map>(); const typedSubjects = new Map>(); @@ -145,6 +148,7 @@ function collectPostingScores( for (const posting of memory.semanticRefIndex.lookupTerm(term)) { const ref = memory.semanticRefs.get(posting.semanticRefOrdinal); const start = ref.range.start.messageOrdinal; + if (!eligible(start)) continue; const end = ref.range.end ? ref.range.end.messageOrdinal - ((ref.range.end.chunkOrdinal ?? 0) === 0 ? 1 : 0) @@ -182,7 +186,7 @@ function collectPostingScores( score + Math.log( 1 + - memory.messages.length / + messageCount / [...matches.values()].filter((set) => set.has(word), ).length, @@ -254,6 +258,99 @@ function parseSourceUri( export class KnowProCorpusIndex implements CorpusIndex { private memory: DocMemory | undefined; + private activeRevisions = new Map(); + + private eligible(ordinal: number): boolean { + const source = parseSourceUri( + this.memory?.messages.get(ordinal)?.metadata.sourceUrl, + ); + return ( + source !== undefined && + this.activeRevisions.get(source.sourceId) === source.revisionId + ); + } + + private eligibleRanges(sourceIds?: ReadonlySet): kp.TextRange[] { + const ranges: kp.TextRange[] = []; + for ( + let ordinal = 0; + this.memory && ordinal < this.memory.messages.length; + ordinal++ + ) { + const source = parseSourceUri( + this.memory.messages.get(ordinal).metadata.sourceUrl, + ); + if ( + !this.eligible(ordinal) || + (sourceIds && (!source || !sourceIds.has(source.sourceId))) + ) + continue; + // TextRangeCollection looks up ranges by their starting message ordinal. + ranges.push({ + start: { messageOrdinal: ordinal }, + end: { messageOrdinal: ordinal + 1 }, + }); + } + return ranges; + } + + private async saveEligibility(): Promise { + await writeRunbookJson( + path.join(this.indexDirectory, "eligibility.json"), + `${JSON.stringify({ + version: 1, + revisions: [...this.activeRevisions], + })}\n`, + ); + } + + private async loadEligibility(): Promise { + this.activeRevisions.clear(); + for (const [sourceId, messages] of this.sourceMessages) + if (messages.length) + this.activeRevisions.set(sourceId, messages.at(-1)!.revisionId); + let value: unknown; + try { + value = JSON.parse( + await readFile( + path.join(this.indexDirectory, "eligibility.json"), + "utf8", + ), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if ( + !value || + typeof value !== "object" || + !("version" in value) || + value.version !== 1 || + !("revisions" in value) || + !Array.isArray(value.revisions) + ) + throw new Error( + "Invalid document revision eligibility; rebuild the derived index", + ); + const active = new Map(); + for (const entry of value.revisions) { + if ( + !Array.isArray(entry) || + entry.length !== 2 || + typeof entry[0] !== "string" || + typeof entry[1] !== "string" || + active.has(entry[0]) || + !this.sourceMessages + .get(entry[0]) + ?.some((message) => message.revisionId === entry[1]) + ) + throw new Error("Invalid document revision eligibility entry"); + active.set(entry[0], entry[1]); + } + if (active.size !== this.sourceMessages.size) + throw new Error("Incomplete document revision eligibility"); + this.activeRevisions = active; + } private sourceMessages = new Map< string, Array<{ @@ -314,6 +411,7 @@ export class KnowProCorpusIndex implements CorpusIndex { }, ); this.refreshSourceMessages(this.memory); + await this.loadEligibility(); if (hasSemanticIndex && this.memory?.messages.length === 0) { throw new Error( `KnowPro semantic index has no messages: '${semanticIndexPath}'`, @@ -379,6 +477,7 @@ export class KnowProCorpusIndex implements CorpusIndex { const symbols = new Set(); for (const posting of memory.semanticRefIndex.lookupTerm(term)) { const ref = memory.semanticRefs.get(posting.semanticRefOrdinal); + if (!this.eligible(ref.range.start.messageOrdinal)) continue; const subject = ref.knowledgeType === "entity" ? (ref.knowledge as EntityKnowledge).name @@ -438,6 +537,7 @@ export class KnowProCorpusIndex implements CorpusIndex { const key = `${source.sourceId}:${source.revisionId}`; const blockOrdinal = ordinals.get(key) ?? 0; ordinals.set(key, blockOrdinal + 1); + if (!this.eligible(messageOrdinal)) continue; if (scores && !scores.has(messageOrdinal)) continue; yield { ...source, @@ -459,12 +559,21 @@ export class KnowProCorpusIndex implements CorpusIndex { terms, requested, request.query !== undefined, + (ordinal) => this.eligible(ordinal), + this.eligibleRanges().reduce( + (count, range) => + count + + range.end!.messageOrdinal - + range.start.messageOrdinal, + 0, + ), ); const followed = this.sourceFollowingScores(request, scores); if (request.query) { const options = kp.createLanguageSearchOptionsTypical(); options.maxMessageMatches = 8; options.maxKnowledgeMatches = 12; + options.messageRangesInScope = this.eligibleRanges(); const result = await memory.searchWithLanguage( request.query ?? request.terms.join(" "), options, @@ -517,7 +626,11 @@ export class KnowProCorpusIndex implements CorpusIndex { const source = parseSourceUri( memory.messages.get(ordinal).metadata.sourceUrl, ); - if (source && entity.sourceIds.includes(source.sourceId)) + if ( + source && + this.eligible(ordinal) && + entity.sourceIds.includes(source.sourceId) + ) preferred.add(ordinal); } } @@ -544,7 +657,11 @@ export class KnowProCorpusIndex implements CorpusIndex { if (!allowed.has(source.sourceId)) return []; const messages = ( this.sourceMessages.get(source.sourceId) ?? [] - ).filter((entry) => entry.revisionId === source.revisionId); + ).filter( + (entry) => + entry.revisionId === source.revisionId && + this.eligible(entry.messageOrdinal), + ); if (messages.some((entry) => scores.has(entry.messageOrdinal))) return []; return messages.length ? [messages] : []; @@ -619,6 +736,10 @@ export class KnowProCorpusIndex implements CorpusIndex { if (documents.length === 0) { await kp.removeConversationData(this.indexDirectory, indexBaseName); this.memory = undefined; + this.activeRevisions.clear(); + await rm(path.join(this.indexDirectory, "eligibility.json"), { + force: true, + }); this.refreshSourceMessages(); await onProgress({ completed: parts.length, @@ -693,10 +814,17 @@ export class KnowProCorpusIndex implements CorpusIndex { docPartCount: parts.length, }); await kp.timeSearchStage("knowpro.index.persist", () => - memory.writeToFile(this.indexDirectory, indexBaseName), + memory.writeToFile(this.indexDirectory, indexBaseName, true), ); this.memory = memory; this.refreshSourceMessages(memory); + this.activeRevisions = new Map( + documents.map((document) => [ + document.source.sourceId, + document.revision.revisionId, + ]), + ); + await this.saveEligibility(); await onProgress({ completed: total, total, @@ -792,9 +920,15 @@ export class KnowProCorpusIndex implements CorpusIndex { docPartCount: parts.length, }); await kp.timeSearchStage("knowpro.index.persist", () => - memory.writeToFile(this.indexDirectory, indexBaseName), + memory.writeToFile(this.indexDirectory, indexBaseName, true), ); this.refreshSourceMessages(this.memory); + for (const document of documents) + this.activeRevisions.set( + document.source.sourceId, + document.revision.revisionId, + ); + await this.saveEligibility(); await onProgress({ completed: total, total, @@ -819,6 +953,31 @@ export class KnowProCorpusIndex implements CorpusIndex { ); } + public async replace( + document: IndexedDocument, + signal: AbortSignal, + onProgress: (progress: JobProgress) => Promise, + ): Promise { + await this.initialize(); + signal.throwIfAborted(); + if ( + !this.sourceMessages + .get(document.source.sourceId) + ?.some( + (message) => + message.revisionId === document.revision.revisionId, + ) + ) { + await this.append([document], signal, onProgress); + } else { + this.activeRevisions.set( + document.source.sourceId, + document.revision.revisionId, + ); + await this.saveEligibility(); + } + } + private async searchCore( query: string, limit: number, @@ -830,6 +989,7 @@ export class KnowProCorpusIndex implements CorpusIndex { const options = kp.createLanguageSearchOptionsTypical(); options.maxMessageMatches = limit; options.maxKnowledgeMatches = limit; + options.messageRangesInScope = this.eligibleRanges(); const result = await kp.timeSearchStage( "knowpro.index.languageSearch", () => @@ -888,6 +1048,7 @@ export class KnowProCorpusIndex implements CorpusIndex { return { matches: [], whyNoAnswer: "The corpus has no index." }; } const options = kp.createLanguageSearchOptionsTypical(); + options.messageRangesInScope = this.eligibleRanges(sourceIds); options.maxMessageMatches = limit; options.maxKnowledgeMatches = limit; const searched = await memory.searchWithLanguage(query, options); @@ -1054,6 +1215,8 @@ export class KnowProCorpusIndex implements CorpusIndex { >(); for (const semanticRef of this.memory.semanticRefs ?? []) { + if (!this.eligible(semanticRef.range.start.messageOrdinal)) + continue; const message = this.memory.messages.get( semanticRef.range.start.messageOrdinal, ); diff --git a/ts/packages/memory/service/src/revisionTextStore.ts b/ts/packages/memory/service/src/revisionTextStore.ts new file mode 100644 index 0000000000..908d45a8ce --- /dev/null +++ b/ts/packages/memory/service/src/revisionTextStore.ts @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID } from "node:crypto"; +import { + lstat, + mkdir, + readFile, + readdir, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { retryFilesystemPublication } from "./filesystemPublication.js"; + +export interface RevisionTextReference { + sha256: string; + bytes: number; +} + +interface TextRevision { + sourceId: string; + revisionId: string; + contentHash: string; + content: string; +} + +export type SerializedTextRevision = Omit< + T, + "content" +> & { + text: RevisionTextReference; +}; + +export type RevisionTextFaultPoint = + | "blob-written" + | "blob-verified" + | "blob-published" + | "manifest-prepared" + | "purge-started" + | "purge-removed"; + +function digest(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +// This cache is derived from immutable revision identities; the manifest alone +// determines which blobs survive collection. It never retains source text. +export class RevisionTextStore { + private readonly references = new Map(); + + public constructor( + private readonly directory: string, + private readonly checkpoint?: ( + point: RevisionTextFaultPoint, + ) => Promise, + ) {} + + private key(revision: Omit): string { + return JSON.stringify([ + revision.sourceId, + revision.revisionId, + revision.contentHash, + ]); + } + + private async ensureDirectory(): Promise { + await mkdir(this.directory, { recursive: true }); + const stat = await lstat(this.directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error("Unsafe revision text directory"); + } + + private async verified( + file: string, + reference: RevisionTextReference, + ): Promise { + const before = await lstat(file); + if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1) + throw new Error("Unsafe revision text blob"); + const bytes = await readFile(file); + const after = await lstat(file); + if ( + before.ino !== after.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + bytes.length !== reference.bytes || + digest(bytes) !== reference.sha256 + ) + throw new Error(`Corrupt revision text blob '${reference.sha256}'`); + return bytes; + } + + public async read( + revision: SerializedTextRevision, + ): Promise { + const { text } = revision; + if ( + !text || + typeof text.sha256 !== "string" || + !/^[a-f0-9]{64}$/.test(text.sha256) || + !Number.isSafeInteger(text.bytes) || + text.bytes < 0 || + "content" in revision + ) + throw new Error( + "Invalid canonical revision text reference; reset/reimport this corpus", + ); + await this.ensureDirectory(); + const bytes = await this.verified( + path.join(this.directory, `${text.sha256}.json`), + text, + ); + const content: unknown = JSON.parse(bytes.toString("utf8")); + if (typeof content !== "string") + throw new Error("Invalid revision text blob"); + this.references.set(this.key(revision), text); + return content; + } + + public async prepare( + revision: T, + ): Promise> { + const { content, ...metadata } = revision; + let text = this.references.get(this.key(revision)); + if (!text) { + const bytes = Buffer.from(JSON.stringify(content), "utf8"); + text = { sha256: digest(bytes), bytes: bytes.length }; + await this.ensureDirectory(); + const file = path.join(this.directory, `${text.sha256}.json`); + try { + await this.verified(file, text); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") + throw error; + const staged = path.join( + this.directory, + `${text.sha256}.${randomUUID()}.tmp`, + ); + try { + await writeFile(staged, bytes, { flag: "wx" }); + await this.checkpoint?.("blob-written"); + await this.verified(staged, text); + await this.checkpoint?.("blob-verified"); + await retryFilesystemPublication(() => + rename(staged, file), + ); + await this.checkpoint?.("blob-published"); + } finally { + await rm(staged, { force: true }); + } + } + this.references.set(this.key(revision), text); + } + return { ...metadata, text }; + } + + public async prepared(): Promise { + await this.checkpoint?.("manifest-prepared"); + } + + public async purge( + sources: readonly { revisions: readonly TextRevision[] }[], + ): Promise { + const retainedKeys = new Set(); + const retainedBlobs = new Set(); + for (const source of sources) + for (const revision of source.revisions) { + const key = this.key(revision); + const reference = this.references.get(key); + if (!reference) + throw new Error( + "Cannot purge text without a verified manifest reference", + ); + retainedKeys.add(key); + retainedBlobs.add(`${reference.sha256}.json`); + } + await this.ensureDirectory(); + for (const key of this.references.keys()) + if (!retainedKeys.has(key)) this.references.delete(key); + for (const entry of await readdir(this.directory, { + withFileTypes: true, + })) { + if (retainedBlobs.has(entry.name)) continue; + if ( + !entry.isFile() || + !/^[a-f0-9]{64}(?:\.json|\.[a-f0-9-]{36}\.tmp)$/.test( + entry.name, + ) + ) + throw new Error( + `Unexpected entry in revision text store: '${entry.name}'`, + ); + await this.checkpoint?.("purge-started"); + await rm(path.join(this.directory, entry.name)); + await this.checkpoint?.("purge-removed"); + } + } +} diff --git a/ts/packages/memory/service/src/types.ts b/ts/packages/memory/service/src/types.ts index 961ec2ca63..45d1a756b6 100644 --- a/ts/packages/memory/service/src/types.ts +++ b/ts/packages/memory/service/src/types.ts @@ -757,6 +757,11 @@ export interface IndexedFragmentSelection { } export interface CorpusIndex { + replace?( + document: IndexedDocument, + signal: AbortSignal, + onProgress: (progress: JobProgress) => Promise, + ): Promise; initialize(): Promise; listSubjects?( offset: number, diff --git a/ts/packages/memory/service/src/viewHistory.ts b/ts/packages/memory/service/src/viewHistory.ts index b08115b0d7..b7530f26b4 100644 --- a/ts/packages/memory/service/src/viewHistory.ts +++ b/ts/packages/memory/service/src/viewHistory.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import git from "isomorphic-git"; +import git, { type TreeEntry } from "isomorphic-git"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import { access, mkdir, readFile, rename, rm } from "node:fs/promises"; @@ -9,6 +9,11 @@ import path from "node:path"; import lockfile from "proper-lockfile"; import { writeRunbookJson } from "./durableRunbookJson.js"; import { retryFilesystemPublication } from "./filesystemPublication.js"; +import { + readViewRecord, + viewRecordsPath, + writeViewRecords, +} from "./viewHistoryRecords.js"; const ref = "refs/heads/typeagent-history"; @@ -25,6 +30,7 @@ async function exists(file: string): Promise { export interface HistorySnapshot { head: string | null; state: T; + stateBytes: number; } export type ViewHistoryFaultPoint = @@ -47,6 +53,7 @@ export class ViewHistory { private readonly checkpoint?: ( point: ViewHistoryFaultPoint, ) => Promise, + private readonly records?: (state: T) => Record, ) { this.gitdir = path.join(directory, "view-history.git"); this.gate = path.join(directory, "view-history-purge.json"); @@ -79,12 +86,14 @@ export class ViewHistory { return { fs: this.filesystem, gitdir: this.gitdir }; } - private async lock(): Promise<() => Promise> { + private async lock(retry = false): Promise<() => Promise> { await mkdir(this.directory, { recursive: true }); return lockfile.lock(this.directory, { realpath: false, lockfilePath: `${this.gitdir}.lock`, - retries: 0, + retries: retry + ? { retries: 20, minTimeout: 10, maxTimeout: 100 } + : 0, }); } @@ -101,7 +110,12 @@ export class ViewHistory { if (!bypassGate) await this.assertReadable(); const head = oid ?? (await this.head()); if (!head) { - return { head: null, state: this.empty() }; + const state = this.empty(); + return { + head: null, + state, + stateBytes: 0, + }; } const blob = await git.readBlob({ ...this.options(), @@ -112,6 +126,7 @@ export class ViewHistory { return { head, state: JSON.parse(Buffer.from(blob.blob).toString("utf8")) as T, + stateBytes: blob.blob.byteLength, }; } @@ -187,6 +202,10 @@ export class ViewHistory { }, ); } + if (this.records) + entries.push( + await writeViewRecords(this.options(), this.records(state)), + ); await this.checkpoint?.("blob"); const tree = await git.writeTree({ ...this.options(), @@ -242,6 +261,102 @@ export class ViewHistory { ); } + public async historyForRecord( + id: string, + expectedHead?: string | null, + ): Promise> { + await this.assertReadable(); + const head = + expectedHead === undefined ? await this.head() : expectedHead; + await this.assertHead(head); + if (!head) return []; + const commits = await git.log({ ...this.options(), ref: head }); + const records = new Map>(); + const trees = new Map>(); + const readTree = (oid: string) => { + let tree = trees.get(oid); + if (!tree) { + tree = git + .readTree({ ...this.options(), oid }) + .then((result) => result.tree); + trees.set(oid, tree); + } + return tree; + }; + const roots = await Promise.all( + commits.map((entry) => readTree(entry.commit.tree)), + ); + const recordTrees: Array> = []; + for (const [index, entry] of commits.entries()) { + const recordEntry = roots[index].find( + (entry) => entry.path === viewRecordsPath, + ); + recordTrees.push( + recordEntry + ? readTree(recordEntry.oid) + : await this.legacyRecordTree(entry.oid, head), + ); + } + const result = await Promise.all( + commits.map(async (entry, index) => ({ + commitId: entry.oid, + record: await readViewRecord( + this.options(), + await recordTrees[index], + id, + records, + ), + })), + ); + await this.assertHead(head); + return result; + } + + private async legacyRecordTree( + commitId: string, + expectedHead: string, + ): Promise { + const cache = path.join(this.gitdir, "view-record-index", commitId); + const cached = await this.cachedRecordTree(cache); + if (cached) return cached; + // Legacy commits are immutable. Rebuild their full-fidelity per-View + // records once, inside the repository removed by the privacy swap. + const release = await this.lock(true); + try { + await this.assertHead(expectedHead); + const cached = await this.cachedRecordTree(cache); + if (cached) return cached; + if (!this.records) + throw new Error("View history record selector is required"); + const { oid } = await writeViewRecords( + this.options(), + this.records((await this.read(commitId)).state), + ); + await writeRunbookJson(cache, oid); + return (await git.readTree({ ...this.options(), oid })).tree; + } finally { + await release(); + } + } + + private async cachedRecordTree( + cache: string, + ): Promise { + try { + const oid = await readFile(cache, "utf8"); + if (/^[a-f0-9]{40}$/.test(oid)) + return (await git.readTree({ ...this.options(), oid })).tree; + } catch (error) { + if ( + !["ENOENT", "NotFoundError", "ObjectTypeError"].includes( + (error as NodeJS.ErrnoException).code ?? "", + ) + ) + throw error; + } + return undefined; + } + public async purge( sourceId: string, sanitize: (state: T, sourceId: string) => T, @@ -288,7 +403,12 @@ export class ViewHistory { (await this.read(undefined, true)).state, sourceId, ); - const fresh = new ViewHistory(replacement, this.empty); + const fresh = new ViewHistory( + replacement, + this.empty, + undefined, + this.records, + ); const snapshot = await fresh.read(); if (!snapshot.head) await fresh.commit( diff --git a/ts/packages/memory/service/src/viewHistoryRecords.ts b/ts/packages/memory/service/src/viewHistoryRecords.ts new file mode 100644 index 0000000000..a860eb04dc --- /dev/null +++ b/ts/packages/memory/service/src/viewHistoryRecords.ts @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import git, { type FsClient, type TreeEntry } from "isomorphic-git"; +import { createHash } from "node:crypto"; + +export const viewRecordsPath = "view-records"; + +export interface ViewRecordGitOptions { + fs: FsClient; + gitdir: string; +} + +function recordPath(id: string): string { + return createHash("sha256").update(id).digest("hex") + ".json"; +} + +export async function writeViewRecords( + options: ViewRecordGitOptions, + records: Record, +): Promise { + const tree: TreeEntry[] = []; + for (const [id, record] of Object.entries(records)) { + tree.push({ + mode: "100644", + path: recordPath(id), + type: "blob", + oid: await git.writeBlob({ + ...options, + blob: Buffer.from(JSON.stringify(record), "utf8"), + }), + }); + } + return { + mode: "040000", + path: viewRecordsPath, + type: "tree", + oid: await git.writeTree({ ...options, tree }), + }; +} + +export async function readViewRecord( + options: ViewRecordGitOptions, + tree: TreeEntry[], + id: string, + records: Map>, +): Promise { + const entry = tree.find((entry) => entry.path === recordPath(id)); + if (!entry) return undefined; + if (records.has(entry.oid)) return records.get(entry.oid); + const record = git + .readBlob({ ...options, oid: entry.oid }) + .then( + (blob) => JSON.parse(Buffer.from(blob.blob).toString("utf8")) as R, + ); + records.set(entry.oid, record); + return record; +} diff --git a/ts/packages/memory/service/src/viewStore.ts b/ts/packages/memory/service/src/viewStore.ts index e67f1b6386..615f0054e3 100644 --- a/ts/packages/memory/service/src/viewStore.ts +++ b/ts/packages/memory/service/src/viewStore.ts @@ -183,11 +183,19 @@ export class TypedViewStore { private readonly actor = "local-owner", ) {} + private readonly topicSizes = new Map< + string, + { head: string | null; bytes: number } + >(); + private history(corpusId: string): ViewHistory { validateIdentifier("corpus ID", corpusId); - return new ViewHistory(this.viewDirectory(corpusId), () => ({ - views: {}, - })); + return new ViewHistory( + this.viewDirectory(corpusId), + () => ({ views: {} }), + undefined, + (state: ViewStoreState) => state.views, + ); } public async withReadSnapshot( @@ -259,7 +267,9 @@ export class TypedViewStore { eligibleFingerprint?: string, ): Promise { const started = performance.now(); - const { state } = await this.history(request.corpusId).read(); + const { head, state, stateBytes } = await this.history( + request.corpusId, + ).read(); if (state.topics) validateTopicSnapshot(state.topics); const withheld = state.topics && @@ -282,11 +292,17 @@ export class TypedViewStore { ...page.limits, `Topic family failure: ${state.topicFamily.error}`, ]; + let size = this.topicSizes.get(request.corpusId); + if (!size || size.head !== head) { + size = { + head, + bytes: Buffer.byteLength(JSON.stringify(state.topics ?? null)), + }; + this.topicSizes.set(request.corpusId, size); + } page.io = { - stateBytes: Buffer.byteLength(JSON.stringify(state)), - snapshotBytes: Buffer.byteLength( - JSON.stringify(state.topics ?? null), - ), + stateBytes, + snapshotBytes: size.bytes, elapsedMs: performance.now() - started, }; return page; @@ -1494,13 +1510,19 @@ export class TypedViewStore { request: ViewReadRequest, ): Promise { assertViewIdentifier("view ID", request.viewId); + const scope = this.readOperation.getStore(); + const head = + scope?.active && scope.corpusId === request.corpusId + ? (await scope.snapshot).head + : undefined; const commits = ( - await this.history(request.corpusId).history() + await this.history(request.corpusId).historyForRecord< + ViewVersion[] + >(request.viewId, head) ).reverse(); const seen = new Set(); - const entries = commits.flatMap(({ commitId, state }) => { - const versions = viewVersions(state, request.viewId); - return versions.flatMap((version) => { + const entries = commits.flatMap(({ commitId, record }) => { + return (record ?? []).flatMap((version) => { if (seen.has(version.revisionId)) return []; seen.add(version.revisionId); return [{ commitId, version }]; diff --git a/ts/packages/memory/service/test/batchIngestion.spec.ts b/ts/packages/memory/service/test/batchIngestion.spec.ts new file mode 100644 index 0000000000..2f3e950286 --- /dev/null +++ b/ts/packages/memory/service/test/batchIngestion.spec.ts @@ -0,0 +1,663 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { mock } from "node:test"; +import { BoundedIngestionGroup } from "../src/boundedIngestionGroup.js"; +import { FileMemoryService } from "../src/fileMemoryService.js"; +import { waitForMemoryJob } from "../src/rpcFacade.js"; +import type { + IndexedDocument, + JobProgress, + DocumentIngestResult, +} from "../src/types.js"; +import type { MemoryBatchImport } from "../src/batchImport.js"; +import { FakeSemanticCorpusIndex } from "./fakeSemanticCorpusIndex.js"; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((finish) => { + resolve = finish; + }); + return { promise, resolve }; +} + +async function within(work: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("Batch fixture did not settle")), + 4000, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +describe("bounded ingestion admission", () => { + test("schedules a lone admission with the exact 25 ms window", async () => { + const timer = mock.method(globalThis, "setTimeout"); + try { + const calls: number[][] = []; + const group = new BoundedIngestionGroup(async (items) => { + calls.push(items); + }); + const work = group.run(async () => { + await group.submit(1); + }); + expect(calls).toEqual([]); + expect(timer.mock.calls[0]?.arguments[1]).toBe(25); + await work; + expect(calls).toEqual([[1]]); + } finally { + timer.mock.restore(); + } + }); + + test("groups exactly two admissions without changing their order", async () => { + const calls: number[][] = []; + const group = new BoundedIngestionGroup(async (items) => { + calls.push(items); + }); + await group.run(async () => { + await Promise.all([ + group.submit(1), + group.submit(2), + group.submit(3), + ]); + }); + expect(calls).toEqual([[1, 2], [3]]); + }); + + test("a lone durable admission makes progress while its peer is still unaccepted", async () => { + const calls: number[][] = []; + const group = new BoundedIngestionGroup(async (items) => { + calls.push(items); + }); + await within( + group.run(async () => { + await group.submit(1); + await group.submit(2); + }), + ); + expect(calls).toEqual([[1], [2]]); + }); + + test("independent group contexts cannot combine", async () => { + const calls: number[][] = []; + const group = new BoundedIngestionGroup(async (items) => { + calls.push(items); + }); + await Promise.all([ + group.run(async () => { + await group.submit(1); + }), + group.run(async () => { + await group.submit(2); + }), + ]); + expect(calls.map((items) => items[0]).sort()).toEqual([1, 2]); + expect(calls.every((items) => items.length === 1)).toBe(true); + }); + + test("dispatch rejection reaches both admissions", async () => { + const group = new BoundedIngestionGroup(async () => { + throw new Error("Expected dispatch failure"); + }); + await expect( + group.run(async () => { + await Promise.all([group.submit(1), group.submit(2)]); + }), + ).rejects.toThrow("Expected dispatch failure"); + }); +}); + +describe("detached coalesced batch ingestion", () => { + let root: string; + let service: FileMemoryService; + let corpusId: string; + let appends: string[][]; + let rebuilds: string[][]; + let hold: + | { + entered: ReturnType; + release: ReturnType; + aborted: ReturnType; + ignoreAbort: boolean; + fail: boolean; + } + | undefined; + let publicationFailure = false; + const releases: Array<() => void> = []; + + class Index extends FakeSemanticCorpusIndex { + private appending = false; + public override async rebuild(documents: IndexedDocument[]) { + if (!this.appending) + rebuilds.push( + documents.map((document) => document.source.sourceId), + ); + await super.rebuild(documents); + } + public override async append( + documents: IndexedDocument[], + signal?: AbortSignal, + progress?: (value: JobProgress) => Promise, + ) { + appends.push(documents.map((document) => document.source.sourceId)); + const pending = hold; + hold = undefined; + if (pending) { + pending.entered.resolve(); + const onAbort = () => { + pending.aborted.resolve(); + if (!pending.ignoreAbort) pending.release.resolve(); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + try { + await pending.release.promise; + } finally { + signal?.removeEventListener("abort", onAbort); + } + if (pending.fail) throw new Error("Shared provider failure"); + if (!pending.ignoreAbort) signal?.throwIfAborted(); + } + // Deliberately write after an ignored abort to prove cleanup drains us. + this.appending = true; + try { + await super.append(documents); + } finally { + this.appending = false; + } + await progress?.({ + completed: documents.length, + total: documents.length, + operation: "append", + message: "Offline append", + }); + } + } + + function open() { + return new FileMemoryService(root, { + sourceSynopses: false, + viewDrafts: false, + informationalViews: false, + indexFactory: (_, directory) => new Index(directory), + revisionTextCheckpoint: async (point) => { + if (publicationFailure && point === "manifest-prepared") { + publicationFailure = false; + throw new Error("Shared publication failure"); + } + }, + }); + } + + beforeEach(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "batch-ingestion-")); + appends = []; + rebuilds = []; + hold = undefined; + publicationFailure = false; + service = open(); + ({ corpusId } = await service.createCorpus("Grouped sources")); + }); + afterEach(async () => { + for (const release of releases.splice(0)) release(); + await service.close(); + await rm(root, { recursive: true, force: true }); + }); + + function block(ignoreAbort = false, fail = false) { + const pending = { + entered: deferred(), + release: deferred(), + aborted: deferred(), + ignoreAbort, + fail, + }; + releases.push(pending.release.resolve); + hold = pending; + return pending; + } + const source = (id: string) => ({ + sourceId: id, + sourceType: "text" as const, + title: id, + text: `Canonical content for ${id}`, + }); + async function ingest(id: string) { + const result = await service.ingestDocument({ + corpusId, + source: source(id), + }); + expect( + ( + await waitForMemoryJob(service, result.jobId, { + pollIntervalMs: 5, + }) + ).state, + ).toBe("complete"); + return result; + } + async function admit(ids = ["first", "second"]) { + let accepted: DocumentIngestResult[] = []; + await service.withIngestionGroup(async () => { + accepted = await Promise.all( + ids.map((id) => + service.ingestDocument({ corpusId, source: source(id) }), + ), + ); + }); + return accepted; + } + const settle = (jobs: DocumentIngestResult[]) => + within( + Promise.all( + jobs.map((job) => + waitForMemoryJob(service, job.jobId, { pollIntervalMs: 5 }), + ), + ), + ); + async function terminal(batch: MemoryBatchImport) { + let polls = 0; + while (["running", "cancelling"].includes(batch.state)) { + if (++polls > 500) + throw new Error( + JSON.stringify({ + batch, + appends, + jobs: await Promise.all( + batch.members.map((member) => + member.jobId + ? service.getJob(member.jobId) + : undefined, + ), + ), + }), + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + batch = await service.getBatchImport(batch.batchId); + } + return batch; + } + async function startBatch(ids = ["first", "second"]) { + return service.startBatchImport({ + corpusId, + idempotencyKey: "offline-batch", + documents: ids.map((id) => ({ source: source(id) })), + }); + } + async function oneGeneration() { + expect(await readdir(path.join(root, corpusId, "index"))).toHaveLength( + 1, + ); + } + + test("eight acquired sources share append generations while preserving eight distinct jobs and content", async () => { + await ingest("seed"); + const batch = await within( + terminal( + await startBatch( + Array.from({ length: 8 }, (_, i) => `member-${i}`), + ), + ), + ); + expect( + batch.members.filter((member) => member.state !== "complete"), + ).toEqual([]); + expect(batch.state).toBe("complete"); + expect(new Set(batch.members.map((member) => member.jobId)).size).toBe( + 8, + ); + expect(appends.length).toBeLessThan(8); + expect(appends.flat().sort()).toEqual( + Array.from({ length: 8 }, (_, i) => `member-${i}`), + ); + expect(appends.every((items) => items.length <= 2)).toBe(true); + for (const member of batch.members) + expect( + ( + await service.getSourceContent({ + corpusId, + sourceId: member.sourceId!, + }) + ).content, + ).toBe(source(member.sourceId!).text); + await oneGeneration(); + }); + + test("an empty corpus builds one two-document generation", async () => { + expect((await settle(await admit())).map((job) => job.state)).toEqual([ + "complete", + "complete", + ]); + expect(rebuilds.map((items) => [...items].sort())).toEqual([ + ["first", "second"], + ]); + expect(appends).toEqual([]); + await oneGeneration(); + }); + + test("ordinary independent ingestions remain singleton operations", async () => { + await ingest("seed"); + await Promise.all([ingest("first"), ingest("second")]); + expect(appends.every((items) => items.length === 1)).toBe(true); + }); + + test("replacement and a new source use the existing separate paths", async () => { + await ingest("first"); + const accepted: DocumentIngestResult[] = []; + await service.withIngestionGroup(async () => { + accepted.push( + await service.ingestDocument({ + corpusId, + source: { ...source("first"), text: "Updated first" }, + pipeline: { updatePolicy: "replaceActiveRevision" }, + }), + ); + accepted.push( + await service.ingestDocument({ + corpusId, + source: source("second"), + }), + ); + }); + expect( + (await settle(accepted)).every((job) => job.state === "complete"), + ).toBe(true); + expect(rebuilds).toEqual([["first"], ["first"]]); + expect(appends).toEqual([["second"]]); + }); + + test("same-source admissions preserve failIfExists semantics instead of merging", async () => { + const jobs: DocumentIngestResult[] = []; + await service.withIngestionGroup(async () => { + jobs.push( + await service.ingestDocument({ + corpusId, + source: source("first"), + }), + ); + jobs.push( + await service.ingestDocument({ + corpusId, + source: source("first"), + pipeline: { updatePolicy: "failIfExists" }, + }), + ); + }); + expect((await settle(jobs)).map((job) => job.state)).toEqual([ + "complete", + "failed", + ]); + expect(await service.listSources(corpusId)).toHaveLength(1); + }); + + test("one invalid member does not fail an eligible peer", async () => { + await ingest("seed"); + const batch = await within( + terminal( + await service.startBatchImport({ + corpusId, + idempotencyKey: "bad-member", + documents: [ + { source: source("first") }, + { + source: { + ...source("second"), + contentHash: "not-the-content-hash", + }, + }, + ], + }), + ), + ); + expect(batch.state).toBe("partial"); + expect(batch.members.map((member) => member.state)).toEqual([ + "complete", + "failed", + ]); + expect(appends).toEqual([["first"]]); + }); + + test("a cancelled member discards the candidate and retries the uncancelled peer after provider writes drain", async () => { + await ingest("seed"); + const pending = block(true); + const jobs = await admit(); + await within(pending.entered.promise); + await service.cancelJob(jobs[0]!.jobId); + await within(pending.aborted.promise); + expect((await service.getJob(jobs[1]!.jobId))!.state).not.toBe( + "complete", + ); + pending.release.resolve(); + expect((await settle(jobs)).map((job) => job.state)).toEqual([ + "cancelled", + "complete", + ]); + expect( + (await service.listSources(corpusId)).map((item) => item.sourceId), + ).toEqual(["seed", "second"]); + expect(appends.map((items) => [...items].sort())).toEqual([ + ["first", "second"], + ["second"], + ]); + await oneGeneration(); + }); + + test("source forgetting drains an abort-ignoring grouped provider before privacy purge and retries unaffected members", async () => { + await ingest("seed"); + const preview = await service.previewForgetSource(corpusId, "seed"); + const pending = block(true); + const jobs = await admit(); + await within(pending.entered.promise); + let forgotten = false; + const removal = service + .forgetSource({ + corpusId, + sourceId: "seed", + confirmationToken: preview.confirmationToken, + }) + .then(() => { + forgotten = true; + }); + await within(pending.aborted.promise); + expect(forgotten).toBe(false); + await expect(service.listSources(corpusId)).rejects.toThrow( + "pending purge recovery", + ); + pending.release.resolve(); + await within(removal); + expect( + (await settle(jobs)).every((job) => job.state === "complete"), + ).toBe(true); + expect( + (await service.listSources(corpusId)) + .map((item) => item.sourceId) + .sort(), + ).toEqual(["first", "second"]); + await oneGeneration(); + }); + + test("a competing reindex discards stale grouped preparation and retries both sources", async () => { + await ingest("seed"); + const pending = block(); + const jobs = await admit(); + await within(pending.entered.promise); + await within(service.reindexCorpus(corpusId)); + pending.release.resolve(); + expect( + (await settle(jobs)).every((job) => job.state === "complete"), + ).toBe(true); + expect(appends[0]).toHaveLength(2); + expect(appends.slice(1).map((items) => items.length)).toEqual([1, 1]); + expect(await service.listSources(corpusId)).toHaveLength(3); + await oneGeneration(); + }); + + test("corpus clear drains the grouped provider without retrying pre-clear admissions", async () => { + await ingest("seed"); + const pending = block(true); + const jobs = await admit(); + await within(pending.entered.promise); + let cleared = false; + const clearing = service.clearCorpus(corpusId).then(() => { + cleared = true; + }); + await within(pending.aborted.promise); + expect(cleared).toBe(false); + pending.release.resolve(); + await within(clearing); + expect((await settle(jobs)).map((job) => job.state)).toEqual([ + "cancelled", + "cancelled", + ]); + expect(await service.listSources(corpusId)).toEqual([]); + expect( + await readdir(path.join(root, corpusId, "revision-text")), + ).toEqual([]); + await oneGeneration(); + }); + + test("shared provider failure is explicit per member and durable retry preserves source identities", async () => { + await ingest("seed"); + const pending = block(false, true); + const started = await startBatch(); + await within(pending.entered.promise).catch(async (error: unknown) => { + throw new Error( + `Provider admission stalled: ${String(error)}; ${JSON.stringify(await service.getBatchImport(started.batchId))}`, + ); + }); + pending.release.resolve(); + const failed = await within(terminal(started)); + const affected = new Set(appends[0]); + expect( + failed.members + .filter((member) => member.state === "failed") + .map((member) => member.sourceId) + .sort(), + ).toEqual([...affected].sort()); + for (const member of failed.members.filter((member) => + affected.has(member.sourceId!), + )) + expect(member.reason).toContain("Shared provider failure"); + expect(await service.listSources(corpusId)).toHaveLength( + 3 - affected.size, + ); + await oneGeneration(); + const retried = await within( + terminal(await service.retryBatchImport(started.batchId)), + ); + expect(retried.state).toBe("complete"); + expect(retried.members.map((member) => member.sourceId)).toEqual( + failed.members.map((member) => member.sourceId), + ); + }); + + test("failed manifest publication exposes neither member and restart collects staged text", async () => { + await ingest("seed"); + publicationFailure = true; + const jobs = await admit(); + expect( + (await settle(jobs)).every( + (job) => + job.state === "failed" && + job.error?.includes("Shared publication failure"), + ), + ).toBe(true); + expect( + (await service.listSources(corpusId)).map((item) => item.sourceId), + ).toEqual(["seed"]); + await service.close(); + service = open(); + await service.initialize(); + expect( + await readdir(path.join(root, corpusId, "revision-text")), + ).toHaveLength(1); + await oneGeneration(); + }); + + test.each([false, true])( + "restart recovers durable batch members across manifest publication (published: %s)", + async (published) => { + await ingest("seed"); + const pending = block(true); + let batch = await startBatch(); + await within(pending.entered.promise); + await within( + (async () => { + while (batch.members.some((member) => !member.jobId)) { + await new Promise((resolve) => setTimeout(resolve, 5)); + batch = await service.getBatchImport(batch.batchId); + } + })(), + ); + const checkpoints = await Promise.all( + [ + path.join(root, "batches", `${batch.batchId}.json`), + ...batch.members.map((member) => + path.join(root, "jobs", `${member.jobId}.json`), + ), + ].map(async (file) => ({ file, bytes: await readFile(file) })), + ); + if (published) { + pending.release.resolve(); + expect((await within(terminal(batch))).state).toBe("complete"); + await service.close(); + } else { + let closed = false; + const closing = service.close().then(() => { + closed = true; + }); + await within(pending.aborted.promise); + expect(closed).toBe(false); + pending.release.resolve(); + await within(closing); + } + // Restore the real pre-completion receipts to model either durable crash + // boundary without leaving a live owner or an undrained provider behind. + for (const checkpoint of checkpoints) + await writeFile(checkpoint.file, checkpoint.bytes); + service = open(); + await service.initialize(); + const recovered = await service.getBatchImport(batch.batchId); + expect(recovered.state).toBe("interrupted"); + expect( + recovered.members.every( + (member) => member.state === "interrupted", + ), + ).toBe(true); + for (const member of recovered.members) + expect((await service.getJob(member.jobId!))?.error).toContain( + "interrupted", + ); + expect(await service.listSources(corpusId)).toHaveLength( + published ? 3 : 1, + ); + const priorAppends = appends.length; + expect( + ( + await within( + terminal(await service.retryBatchImport(batch.batchId)), + ) + ).state, + ).toBe("complete"); + if (published) expect(appends).toHaveLength(priorAppends); + for (const id of ["first", "second"]) + expect( + (await service.getSourceContent({ corpusId, sourceId: id })) + .content, + ).toBe(source(id).text); + await oneGeneration(); + }, + ); +}); diff --git a/ts/packages/memory/service/test/eventKnowPro.spec.ts b/ts/packages/memory/service/test/eventKnowPro.spec.ts index db5b630a83..f6db2606fc 100644 --- a/ts/packages/memory/service/test/eventKnowPro.spec.ts +++ b/ts/packages/memory/service/test/eventKnowPro.spec.ts @@ -158,7 +158,7 @@ test("event search uses structured knowledge with ledger filters before the limi test("event index reconciles replay, failed extraction, forgetting, and restart", async () => { const fixture = await createFixture(); - const { root, failExtraction } = fixture; + const { root, failExtraction, extracted } = fixture; let service = fixture.service; try { const { corpusId } = await service.createCorpus("Event lifecycle"); @@ -177,9 +177,10 @@ test("event index reconciles replay, failed extraction, forgetting, and restart" (await service.searchEvents({ corpusId, query: question })) .matches[0].event.eventId, ).toBe(first.event.eventId); + expect(extracted).toHaveLength(1); const indexRoot = path.join(root, corpusId, "event-search-index"); - expect(await readdir(indexRoot)).toHaveLength(2); + expect(await readdir(indexRoot)).toHaveLength(3); const second = await service.appendEvent({ ...request, idempotencyKey: "second", @@ -189,15 +190,17 @@ test("event index reconciles replay, failed extraction, forgetting, and restart" await expect( service.searchEvents({ corpusId, query: question }), ).rejects.toThrow("Expected extraction failure"); - expect(await readdir(indexRoot)).toEqual([]); + expect(await readdir(indexRoot)).toHaveLength(3); expect((await service.listEvents({ corpusId })).total).toBe(2); failExtraction.value = false; + const beforeAppend = extracted.length; expect( (await service.searchEvents({ corpusId, query: question })).matches, ).toHaveLength(2); + expect(extracted.length - beforeAppend).toBe(1); const generation = (await readdir(indexRoot)).find( - (entry) => entry !== "state.json", + (entry) => entry !== "state.json" && entry !== ".segments", ); expect(generation).toBeDefined(); await rm(path.join(indexRoot, generation!), { @@ -210,9 +213,11 @@ test("event index reconciles replay, failed extraction, forgetting, and restart" await service.close(); service = fixture.makeService(); + const beforeRestart = extracted.length; expect( (await service.searchEvents({ corpusId, query: question })).matches, ).toHaveLength(2); + expect(extracted).toHaveLength(beforeRestart); expect((await service.appendEvent(request)).replayed).toBe(true); await service.forgetEvents({ @@ -238,7 +243,7 @@ test("event index reconciles replay, failed extraction, forgetting, and restart" await expect(service.appendEvent(request)).rejects.toMatchObject({ code: "EVENT_FORGOTTEN", }); - expect(await readdir(indexRoot)).toHaveLength(2); + expect(await readdir(indexRoot)).toHaveLength(3); await service.forgetEvents({ corpusId, eventIds: [second.event.eventId], @@ -295,6 +300,7 @@ async function createFixture() { const previousProvider = process.env.TYPEAGENT_EMBEDDING_PROVIDER; process.env.TYPEAGENT_EMBEDDING_PROVIDER = "none"; const failExtraction = { value: false }; + const extracted: string[] = []; const knowledgeForText = (text: string) => ({ entities: text.includes("Zephyr") ? [{ name: "Zephyr", type: ["service"] }] @@ -340,8 +346,9 @@ async function createFixture() { settings: { maxContextLength: 1000 }, extract: async (text: string) => knowledgeForText(text), - extractWithRetry: async (text: string) => - failExtraction.value + extractWithRetry: async (text: string) => { + extracted.push(text); + return failExtraction.value ? { success: false as const, message: @@ -350,7 +357,8 @@ async function createFixture() { : { success: true as const, data: knowledgeForText(text), - }, + }; + }, }; return settings; }), @@ -360,6 +368,7 @@ async function createFixture() { makeService, service: makeService(), failExtraction, + extracted, restoreProvider: () => { if (previousProvider === undefined) { delete process.env.TYPEAGENT_EMBEDDING_PROVIDER; diff --git a/ts/packages/memory/service/test/eventSegmentStore.spec.ts b/ts/packages/memory/service/test/eventSegmentStore.spec.ts new file mode 100644 index 0000000000..861cba4551 --- /dev/null +++ b/ts/packages/memory/service/test/eventSegmentStore.spec.ts @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { + mkdtemp, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { + EventSegmentStore, + type EventStorageFaultPoint, +} from "../src/eventSegmentStore.js"; +import type { MemoryEvent } from "../src/types.js"; + +describe("segmented canonical event storage", () => { + let root: string; + let file: string; + let failure: EventStorageFaultPoint | undefined; + const event = (id: string, content = `Content ${id}`): MemoryEvent => ({ + eventId: id, + corpusId: "corpus", + idempotencyKey: id, + producer: { producerId: "fixture", producerType: "test" }, + eventType: "turn.completed", + sourceKind: "conversation", + content, + observedAt: "2026-10-19T00:00:00Z", + eventTime: "2026-10-19T00:00:00Z", + createdAt: "2026-10-19T00:00:00Z", + }); + const open = () => + new EventSegmentStore<{ key: string }>( + file, + async () => { + await rm(path.join(root, "derived-private.json"), { + force: true, + }); + }, + async (point) => { + if (point === failure) + throw new Error(`Expected ${point} interruption`); + }, + ); + beforeEach(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "event-segments-")); + file = path.join(root, "events.jsonl"); + failure = undefined; + }); + afterEach(async () => { + await rm(root, { recursive: true, force: true }); + }); + const segment = async (id: string) => { + const identity = createHash("sha256").update(id).digest("hex"); + return path.join( + root, + "event-segments", + (await readdir(path.join(root, "event-segments"))).find((name) => + name.includes(identity), + )!, + ); + }; + + test("append writes one new immutable event and a bounded pointer, not retained payloads or lookup maps", async () => { + const store = open(); + for (let i = 0; i < 100; i++) + await store.append(event(`event-${i}`, "Retained ".repeat(1000))); + const prior = await stat(await segment("event-0")); + const before = await readdir(path.join(root, "event-segments")); + await store.append(event("new", "Changed content")); + expect( + (await readdir(path.join(root, "event-segments"))).length - + before.length, + ).toBe(1); + expect((await stat(file)).size).toBeLessThan(160); + const retained = await stat(await segment("event-0")); + expect(retained.ino).toBe(prior.ino); + expect(retained.mtimeMs).toBe(prior.mtimeMs); + expect((await open().get("new"))?.content).toBe("Changed content"); + }); + + test("cold selective lookup does not read or decode unrelated event payloads", async () => { + const store = open(); + await store.append(event("wanted")); + await store.append( + event("unrelated", "Very large retained content ".repeat(2000)), + ); + await writeFile(await segment("unrelated"), "Corrupt unrelated body"); + expect(await open().get("wanted")).toEqual(event("wanted")); + await expect(open().records()).rejects.toThrow("Corrupt"); + }); + + test("deletion changes eligibility without rewinding sequence and cannot restore a forgotten identity", async () => { + const store = open(); + await store.append(event("first")); + await store.append(event("second")); + await store.replace([event("second")], [{ key: "first-suppressed" }]); + expect(await store.version()).toEqual({ + sequence: 2, + eligibilityRevision: 1, + }); + await store.append(event("third")); + expect(await store.version()).toEqual({ + sequence: 3, + eligibilityRevision: 1, + }); + expect(await open().get("first")).toBeUndefined(); + expect( + (await open().records()).events.map((value) => value.eventId), + ).toEqual(["second", "third"]); + await expect(store.append(event("first"))).rejects.toThrow("Duplicate"); + }); + + test.each(["segment-written", "descriptor-prepared"] as const)( + "%s append interruption preserves the committed prefix", + async (point) => { + const store = open(); + await store.append(event("first")); + const before = await readFile(file, "utf8"); + failure = point; + await expect( + store.append(event("second", "Uncommitted private payload")), + ).rejects.toThrow("Expected"); + expect(await readFile(file, "utf8")).toBe(before); + failure = undefined; + expect((await open().records()).events).toEqual([event("first")]); + expect( + await readdir(path.join(root, "event-segments")), + ).toHaveLength(1); + }, + ); + + test.each(["purge-started", "purge-finished"] as const)( + "%s interruption withholds reads and resumes physical purge", + async (point) => { + const store = open(); + await store.append(event("private", "Forgotten private payload")); + await store.append(event("public", "Retained public payload")); + await writeFile( + path.join(root, "derived-private.json"), + "Forgotten private payload", + ); + failure = point; + await expect( + store.replace( + [event("public", "Retained public payload")], + [{ key: "private-suppressed" }], + ), + ).rejects.toThrow("Expected"); + await expect(store.get("private")).rejects.toThrow("Expected"); + failure = undefined; + const recovered = open(); + await recovered.recover(); + expect(await recovered.get("private")).toBeUndefined(); + expect(await recovered.get("public")).toEqual( + event("public", "Retained public payload"), + ); + expect( + JSON.parse(await readFile(file, "utf8")).pendingPurge, + ).toBeUndefined(); + for (const name of await readdir(path.join(root, "event-segments"))) + expect( + await readFile( + path.join(root, "event-segments", name), + "utf8", + ), + ).not.toContain("Forgotten private payload"); + await expect( + stat(path.join(root, "derived-private.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + test("missing committed segments and unsupported canonical formats fail without resetting data", async () => { + const store = open(); + await store.append(event("first")); + await rm(await segment("first")); + await expect(open().get("first")).rejects.toThrow("Missing committed"); + const incompatible = '{"eventStorageVersion":99,"preserve":"original"}'; + await writeFile(file, incompatible); + await expect(open().recover()).rejects.toThrow("reset/reimport"); + expect(await readFile(file, "utf8")).toBe(incompatible); + }); +}); diff --git a/ts/packages/memory/service/test/fileMemoryService.spec.ts b/ts/packages/memory/service/test/fileMemoryService.spec.ts index b16fa94d94..7af002fc35 100644 --- a/ts/packages/memory/service/test/fileMemoryService.spec.ts +++ b/ts/packages/memory/service/test/fileMemoryService.spec.ts @@ -37,6 +37,7 @@ class FakeCorpusIndex implements CorpusIndex { public blockNextRebuild = false; public blockNextAppend = false; public ignoreNextAbort = false; + public releaseIgnoredRebuild?: () => void; public answer?: NonNullable; public graph: MemoryKnowledgeGraph = { entities: [], @@ -66,6 +67,7 @@ class FakeCorpusIndex implements CorpusIndex { await new Promise((resolve, reject) => { if (this.ignoreNextAbort) { this.ignoreNextAbort = false; + this.releaseIgnoredRebuild = resolve; return; } if (signal.aborted) { @@ -205,6 +207,7 @@ describe("FileMemoryService", () => { }); afterEach(async () => { + index.releaseIgnoredRebuild?.(); await service.close(); await rm(rootDirectory, { recursive: true, force: true }); }); @@ -742,10 +745,18 @@ describe("FileMemoryService", () => { ...event("verified", "conversation-2", "authority-turn"), metadata: { authority: "verified-observation" }, }); - const ledger = await readFile( - path.join(rootDirectory, corpusId, "events.jsonl"), - "utf8", + const eventSegments = path.join( + rootDirectory, + corpusId, + "event-segments", ); + const ledger = ( + await Promise.all( + (await readdir(eventSegments)).map((file) => + readFile(path.join(eventSegments, file), "utf8"), + ), + ) + ).join("\n"); expect(ledger).toContain('"recordType":"event-suppression"'); expect(ledger).not.toContain("Private removed content"); expect(ledger).not.toContain("Private retained content"); @@ -1796,7 +1807,7 @@ describe("FileMemoryService", () => { expect(index.documents).toHaveLength(0); }); - test("cancels when an index rebuild does not observe the signal", async () => { + test("keeps cancellation pending until an abort-ignoring provider drains without publication", async () => { const corpus = await service.createCorpus("Engineering"); index.blockNextRebuild = true; index.ignoreNextAbort = true; @@ -1821,10 +1832,14 @@ describe("FileMemoryService", () => { await new Promise((resolve) => setTimeout(resolve, 10)); } await service.cancelJob(accepted.jobId); - + expect((await service.getJob(accepted.jobId))?.state).toBe( + "cancelling", + ); + index.releaseIgnoredRebuild!(); expect((await waitForTerminalJob(service, accepted.jobId)).state).toBe( "cancelled", ); + expect(await service.listSources(corpus.corpusId)).toEqual([]); }); test("round trips deterministic procedure Markdown with free-form sections", () => { diff --git a/ts/packages/memory/service/test/markdownPersistence.spec.ts b/ts/packages/memory/service/test/markdownPersistence.spec.ts index 0947702115..78a9245a67 100644 --- a/ts/packages/memory/service/test/markdownPersistence.spec.ts +++ b/ts/packages/memory/service/test/markdownPersistence.spec.ts @@ -201,6 +201,36 @@ describe("Markdown source persistence with stubbed KnowPro", () => { ).toBe(true); } + test("coalesced KnowPro append preserves extraction scope and stable message citations through restart", async () => { + const { corpusId } = await service.createCorpus( + "Grouped Markdown reports", + ); + await ingest(corpusId, "seed"); + const perSourceExtractions = extracted.length; + await service.withIngestionGroup(async () => { + await Promise.all([ + ingest(corpusId, "first"), + ingest(corpusId, "second"), + ]); + }); + await checkEvidence(corpusId, ["seed", "first", "second"]); + expect(extracted.length).toBe(perSourceExtractions * 3); + const citations = async () => + ( + await service.search({ corpusId, query: "Zephyr", limit: 20 }) + ).matches.map((hit) => ({ + sourceId: hit.sourceId, + locator: hit.locator, + snippet: hit.snippet, + })); + const before = await citations(); + await service.close(); + service = openService(); + await checkEvidence(corpusId, ["seed", "first", "second"]); + expect(await citations()).toEqual(before); + expect(extracted.length).toBe(perSourceExtractions * 3); + }); + test("uses ordinary Markdown and retains source attribution through append, restart, and reindex", async () => { const { corpusId } = await service.createCorpus("Markdown reports"); await ingest(corpusId, "first"); @@ -228,6 +258,126 @@ describe("Markdown source persistence with stubbed KnowPro", () => { ).not.toContain("document-projection.json"); }); + test("delta replacement extracts only changed content, filters tombstones before limits, and keeps retained citations stable", async () => { + const { corpusId } = await service.createCorpus( + "Delta Markdown reports", + ); + for (const id of ["first", "unchanged", "also-unchanged"]) + await ingest(corpusId, id); + const citations = async () => + ( + await service.search({ corpusId, query: "Zephyr", limit: 20 }) + ).matches + .map((hit) => ({ + sourceId: hit.sourceId, + revisionId: hit.revisionId, + locator: hit.locator, + snippet: hit.snippet, + })) + .sort((left, right) => + left.sourceId.localeCompare(right.sourceId), + ); + const retained = (await citations()).filter( + (hit) => hit.sourceId !== "first", + ); + const replace = async (text: string) => { + const current = (await service.getSource(corpusId, "first"))!; + const job = await service.replaceSource({ + corpusId, + sourceId: "first", + expectedActiveRevisionId: current.activeRevisionId, + source: { + sourceType: "markdown", + title: "Changing report", + markdown: text, + tags: ["changing"], + }, + }); + await waitForJob(service, job.jobId); + }; + for (let revision = 0; revision < 8; revision++) { + const before = extracted.length; + await replace( + `# Zephyr\n\nZephyr private-delta-${revision} recovery.`, + ); + expect(extracted.length - before).toBe(1); + const result = await service.search({ + corpusId, + query: "Zephyr", + tags: ["changing"], + limit: 1, + }); + expect(result.matches).toHaveLength(1); + expect(result.matches[0]?.snippet).toContain( + `private-delta-${revision}`, + ); + } + expect( + (await citations()).filter((hit) => hit.sourceId !== "first"), + ).toEqual(retained); + const beforeRestart = await citations(); + const fragments = await service.search({ + corpusId, + query: "Zephyr", + indexedFragments: true, + tags: ["changing"], + limit: 1, + }); + expect(fragments.matches).toHaveLength(1); + expect(fragments.matches[0].snippet).toContain("private-delta-7"); + const graph = await service.getKnowledgeGraph(corpusId); + expect( + graph.entities.find((entity) => entity.name === "Zephyr") + ?.mentionCount, + ).toBe(3); + expect( + graph.topics.find((topic) => topic.name === "recovery") + ?.mentionCount, + ).toBe(3); + const extractionCount = extracted.length; + await service.close(); + service = openService(); + expect(await citations()).toEqual(beforeRestart); + expect(extracted).toHaveLength(extractionCount); + await replace(markdown); + expect(extracted).toHaveLength(extractionCount); + expect( + (await service.getSource(corpusId, "first"))?.activeRevisionId, + ).toBe(contentHash); + await service.close(); + service = openService(); + expect( + (await citations()).find((hit) => hit.sourceId === "first") + ?.locator, + ).toBe("message:0"); + const preview = await service.previewForgetSource(corpusId, "first"); + await service.forgetSource({ + corpusId, + sourceId: "first", + confirmationToken: preview.confirmationToken, + }); + for (const directory of ["index", "revision-text"]) { + const rootDirectory = path.join(root, corpusId, directory); + const inspect = async (folder: string): Promise => { + for (const entry of await readdir(folder, { + withFileTypes: true, + })) { + const file = path.join(folder, entry.name); + if (entry.isDirectory()) await inspect(file); + else + expect(await readFile(file, "utf8")).not.toContain( + "private-delta-", + ); + } + }; + await inspect(rootDirectory); + } + expect((await citations()).map((hit) => hit.sourceId)).toEqual([ + "also-unchanged", + "unchanged", + ]); + }); + test.each(["search", "append"])( "%s rebuilds a legacy mapped generation without rewriting retained revisions", async (operation) => { @@ -273,6 +423,7 @@ describe("Markdown source persistence with stubbed KnowPro", () => { expect(migrated.indexGeneration).not.toBe(manifest.indexGeneration); expect(migrated.sources[0].revisions).toEqual(retainedRevisions); expect(await readdir(path.join(root, corpusId, "index"))).toEqual([ + ".segments", migrated.indexGeneration, ]); expect( diff --git a/ts/packages/memory/service/test/readConcurrency.spec.ts b/ts/packages/memory/service/test/readConcurrency.spec.ts index a50adae7dc..b6a8e3bbe1 100644 --- a/ts/packages/memory/service/test/readConcurrency.spec.ts +++ b/ts/packages/memory/service/test/readConcurrency.spec.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { FileMemoryServiceOptions } from "../src/fileMemoryService.js"; @@ -73,6 +73,8 @@ describe("snapshot-safe indexed reads", () => { } | undefined; let rebuilds: number; + let rebuildBlock: ReturnType | undefined; + let failRebuild: boolean; let searchedDirectories: Record; beforeEach(async () => { @@ -86,6 +88,8 @@ describe("snapshot-safe indexed reads", () => { blocks = {}; releases = []; rebuilds = 0; + rebuildBlock = undefined; + failRebuild = false; searchedDirectories = { documents: [], procedures: [] }; failInitialization = undefined; initializationBlock = undefined; @@ -108,6 +112,16 @@ describe("snapshot-safe indexed reads", () => { } public override async rebuild(documents: IndexedDocument[]) { rebuilds++; + const gate = rebuildBlock; + rebuildBlock = undefined; + if (gate) { + gate.entered.resolve(); + await gate.resume.promise; + } + if (failRebuild) { + failRebuild = false; + throw new Error("Expected rebuild failure"); + } await super.rebuild(documents); } public override async search( @@ -160,29 +174,216 @@ describe("snapshot-safe indexed reads", () => { return service.search({ corpusId, query: "Zephyr" }); } - test("reindex publication waits for readers and later readers wait behind the writer", async () => { + function blockRebuild() { + const gate = { entered: deferred(), resume: deferred() }; + rebuildBlock = gate; + releases.push(gate.resume.resolve); + return gate; + } + + test("reindex prepares without blocking reads and publication waits for readers", async () => { + const preparation = blockRebuild(); + const reindex = service.reindexCorpus(corpusId); + await within(preparation.entered.promise); + expect((await within(documentSearch())).matches).toHaveLength(1); + const oldDirectory = searchedDirectories.documents.at(-1); const reader = block("documents"); const first = documentSearch(); await within(reader.entered.promise); - const firstDirectory = searchedDirectories.documents.at(-1); - const beforeRebuilds = rebuilds; - const beforeSearches = counts.documents.searched; - const reindex = service.reindexCorpus(corpusId); - await new Promise((resolve) => setImmediate(resolve)); - const later = documentSearch(); + let published = false; + void reindex.then(() => { + published = true; + }); + preparation.resume.resolve(); await new Promise((resolve) => setImmediate(resolve)); - expect(rebuilds).toBe(beforeRebuilds); - expect(counts.documents.searched).toBe(beforeSearches); + expect(published).toBe(false); reader.resume.resolve(); const firstResult = await first; const publication = await reindex; - const laterResult = await later; + const laterResult = await documentSearch(); expect(firstResult.indexVersion).toBe(publication.indexVersion); - expect(searchedDirectories.documents.at(-1)).not.toBe(firstDirectory); + expect(searchedDirectories.documents.at(-1)).not.toBe(oldDirectory); expect(laterResult.indexVersion).toBe(publication.indexVersion); expect(laterResult.matches).toHaveLength(1); }); + test("failed reindex preserves the healthy generation", async () => { + await documentSearch(); + const directory = searchedDirectories.documents.at(-1); + failRebuild = true; + await expect(service.reindexCorpus(corpusId)).rejects.toThrow( + "Expected rebuild failure", + ); + expect((await documentSearch()).matches).toHaveLength(1); + expect(searchedDirectories.documents.at(-1)).toBe(directory); + expect(await readdir(path.join(root, corpusId, "index"))).toHaveLength( + 1, + ); + }); + + test("concurrent ingestion prevents stale reindex publication", async () => { + const preparation = blockRebuild(); + const reindex = service.reindexCorpus(corpusId); + const rejected = expect(reindex).rejects.toThrow( + "changed during reindex preparation", + ); + await within(preparation.entered.promise); + const accepted = await service.ingestDocument({ + corpusId, + source: { + sourceId: "second", + sourceType: "text", + title: "Second", + text: "Zephyr second", + }, + }); + await waitForJob(service, accepted.jobId); + preparation.resume.resolve(); + await rejected; + expect((await documentSearch()).matches).toHaveLength(2); + expect(await readdir(path.join(root, corpusId, "index"))).toHaveLength( + 1, + ); + }); + + test("forget waits for detached provider writes and never republishes forgotten text", async () => { + const preview = await service.previewForgetSource(corpusId, "source"); + const preparation = blockRebuild(); + const reindex = service.reindexCorpus(corpusId); + const stopped = expect(reindex).rejects.toThrow(); + await within(preparation.entered.promise); + const removalBuild = blockRebuild(); + const forget = service.forgetSource({ + corpusId, + sourceId: "source", + confirmationToken: preview.confirmationToken, + }); + await within(removalBuild.entered.promise); + removalBuild.resume.resolve(); + let finished = false; + void forget.then(() => { + finished = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(finished).toBe(false); + preparation.resume.resolve(); + await stopped; + await forget; + expect((await documentSearch()).matches).toEqual([]); + expect(await readdir(path.join(root, corpusId, "index"))).toHaveLength( + 1, + ); + await service.close(); + service = new FileMemoryService(root, options); + expect((await documentSearch()).matches).toEqual([]); + }); + + test("clear drains detached rebuilds without restoring cleared documents after restart", async () => { + const preparation = blockRebuild(); + const reindex = service.reindexCorpus(corpusId); + const stopped = expect(reindex).rejects.toThrow(); + await within(preparation.entered.promise); + const empty = blockRebuild(); + const clear = service.clearCorpus(corpusId); + await within(empty.entered.promise); + empty.resume.resolve(); + preparation.resume.resolve(); + expect(await clear).toBe(1); + await stopped; + await service.close(); + service = new FileMemoryService(root, options); + expect((await documentSearch()).matches).toHaveLength(0); + expect(await service.listSources(corpusId)).toHaveLength(0); + }); + + test("cancelled reindex waits for provider writes and preserves the active generation", async () => { + const preparation = blockRebuild(); + const controller = new AbortController(); + const reindex = service.reindexCorpus(corpusId, controller.signal); + const stopped = expect(reindex).rejects.toThrow(); + await within(preparation.entered.promise); + controller.abort(); + expect((await within(documentSearch())).matches).toHaveLength(1); + preparation.resume.resolve(); + await stopped; + expect((await documentSearch()).matches).toHaveLength(1); + }); + + test("close cancels publication and waits for detached provider writes", async () => { + const preparation = blockRebuild(); + const reindex = service.reindexCorpus(corpusId); + const stopped = expect(reindex).rejects.toThrow("closing"); + await within(preparation.entered.promise); + let finished = false; + const closing = service.close().then(() => { + finished = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(finished).toBe(false); + preparation.resume.resolve(); + await stopped; + await closing; + expect(await readdir(path.join(root, corpusId, "index"))).toHaveLength( + 1, + ); + }); + + test("concurrent read repair coalesces provider work without holding the write reservation", async () => { + await service.close(); + const indexRoot = path.join(root, corpusId, "index"); + const [generation] = await readdir(indexRoot); + await rm(path.join(indexRoot, generation, "index-schema.json")); + service = new FileMemoryService(root, options); + const preparation = blockRebuild(); + const before = rebuilds; + const first = documentSearch(); + await within(preparation.entered.promise); + const second = documentSearch(); + try { + await within( + service.appendEvent({ + corpusId, + idempotencyKey: "during-repair", + producer: { producerId: "test", producerType: "test" }, + eventType: "turn.completed", + sourceKind: "conversation", + observedAt: "2026-10-19T00:00:00Z", + content: "Metadata write during document repair", + }), + ); + expect(rebuilds).toBe(before + 1); + } finally { + preparation.resume.resolve(); + await Promise.allSettled([first, second]); + } + expect((await first).matches).toHaveLength(1); + expect((await second).matches).toHaveLength(1); + expect(rebuilds).toBe(before + 1); + }); + + test("failed shared repair is released so a later read can retry", async () => { + await service.close(); + const indexRoot = path.join(root, corpusId, "index"); + const [generation] = await readdir(indexRoot); + await rm(path.join(indexRoot, generation, "index-schema.json")); + service = new FileMemoryService(root, options); + failRebuild = true; + const preparation = blockRebuild(); + const first = documentSearch(); + const firstFailure = expect(first).rejects.toThrow( + "Expected rebuild failure", + ); + await within(preparation.entered.promise); + const second = documentSearch(); + const secondFailure = expect(second).rejects.toThrow( + "Expected rebuild failure", + ); + await new Promise((resolve) => setImmediate(resolve)); + preparation.resume.resolve(); + await Promise.all([firstFailure, secondFailure]); + expect((await documentSearch()).matches).toHaveLength(1); + }); + test("forget never changes an in-flight snapshot or resurrects removed evidence", async () => { const preview = await service.previewForgetSource(corpusId, "source"); const reader = block("documents"); diff --git a/ts/packages/memory/service/test/revisionTextStore.spec.ts b/ts/packages/memory/service/test/revisionTextStore.spec.ts new file mode 100644 index 0000000000..7789bd8f7b --- /dev/null +++ b/ts/packages/memory/service/test/revisionTextStore.spec.ts @@ -0,0 +1,423 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { + link, + mkdtemp, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { FileMemoryService } from "../src/fileMemoryService.js"; +import { waitForMemoryJob } from "../src/rpcFacade.js"; +import { + RevisionTextStore, + type RevisionTextFaultPoint, +} from "../src/revisionTextStore.js"; +import { FakeSemanticCorpusIndex } from "./fakeSemanticCorpusIndex.js"; + +interface StoredManifest { + storageVersion: number; + pendingCorpusClear?: unknown; + authorizedSourceRemoval?: unknown; + sources: Array<{ + sourceId: string; + revisions: Array<{ + revisionId: string; + content?: string; + text: { sha256: string; bytes: number }; + }>; + }>; +} + +describe("canonical revision text", () => { + let root: string; + let service: FileMemoryService; + let corpusId: string; + let fault: RevisionTextFaultPoint | undefined; + + function open() { + return new FileMemoryService(root, { + sourceSynopses: false, + viewDrafts: false, + informationalViews: false, + indexFactory: (_, directory) => + new FakeSemanticCorpusIndex(directory), + revisionTextCheckpoint: async (point) => { + if (fault === point) { + fault = undefined; + throw new Error(`Interrupted at ${point}`); + } + }, + }); + } + + beforeEach(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "revision-text-")); + fault = undefined; + service = open(); + ({ corpusId } = await service.createCorpus("Text storage")); + }); + + afterEach(async () => { + await service.close(); + await rm(root, { recursive: true, force: true }); + }); + + async function restart() { + await service.close(); + service = open(); + await service.initialize(); + } + + async function ingest(sourceId: string, text: string, retain = false) { + const accepted = await service.ingestDocument({ + corpusId, + source: { sourceId, sourceType: "text", title: sourceId, text }, + pipeline: { + updatePolicy: retain + ? "retainRevisionHistory" + : "replaceActiveRevision", + }, + }); + return waitForMemoryJob(service, accepted.jobId, { pollIntervalMs: 1 }); + } + + async function seed(sourceId: string, text: string, retain = false) { + const job = await ingest(sourceId, text, retain); + expect(job.state).toBe("complete"); + expect(job.error).toBeUndefined(); + return job; + } + + const manifestFile = () => path.join(root, corpusId, "manifest.json"); + const textDirectory = () => path.join(root, corpusId, "revision-text"); + const manifest = async () => + JSON.parse(await readFile(manifestFile(), "utf8")) as StoredManifest; + const content = (sourceId: string, revisionId?: string) => + service.getSourceContent({ + corpusId, + sourceId, + ...(revisionId === undefined ? {} : { revisionId }), + maxChars: 2000, + }); + + async function forget(sourceId: string) { + const preview = await service.previewForgetSource(corpusId, sourceId); + return service.forgetSource({ + corpusId, + sourceId, + confirmationToken: preview.confirmationToken, + }); + } + + async function assertNoBytes( + directory: string, + secret: string, + ): Promise { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + if (entry.isDirectory()) await assertNoBytes(file, secret); + else + expect( + (await readFile(file)).includes(Buffer.from(secret)), + ).toBe(false); + } + } + + test("stores verified JSON text outside metadata and preserves exact UTF-16 content across restart", async () => { + const text = + 'retained "quotes" \\ slash\n\uD800 lone \uDFFF \u{1F600} end'; + await seed("first", text); + const stored = await manifest(); + expect(stored.storageVersion).toBe(1); + const revision = stored.sources[0].revisions[0]; + expect(revision).not.toHaveProperty("content"); + expect(await readFile(manifestFile(), "utf8")).not.toContain("quotes"); + const bytes = await readFile( + path.join(textDirectory(), `${revision.text.sha256}.json`), + ); + expect(bytes.length).toBe(revision.text.bytes); + expect(createHash("sha256").update(bytes).digest("hex")).toBe( + revision.text.sha256, + ); + expect(JSON.parse(bytes.toString("utf8"))).toBe(text); + await restart(); + expect((await content("first")).content).toBe(text); + expect( + (await service.getSource(corpusId, "first"))?.revisions[0], + ).not.toHaveProperty("text"); + }); + + test("one-source replacement never rewrites retained blobs", async () => { + await seed("retained", "unchanged text"); + await seed("changed", "old text"); + const retained = (await manifest()).sources.find( + (source) => source.sourceId === "retained", + )!; + const file = path.join( + textDirectory(), + `${retained.revisions[0].text.sha256}.json`, + ); + const before = await stat(file); + await seed("changed", "replacement text"); + const after = await stat(file); + expect(after.ino).toBe(before.ino); + expect(after.mtimeMs).toBe(before.mtimeMs); + expect(await readdir(textDirectory())).toHaveLength(2); + await restart(); + expect((await content("retained")).content).toBe("unchanged text"); + expect((await content("changed")).content).toBe("replacement text"); + }); + + test("deduplicates shared content across sources and retained revisions until the final reference is forgotten", async () => { + const shared = "private shared revision bytes"; + await seed("first", shared); + await seed("second", shared); + const original = (await service.getSource(corpusId, "first"))! + .activeRevisionId; + await seed("first", "new first text", true); + expect(await readdir(textDirectory())).toHaveLength(2); + await forget("second"); + expect((await content("first", original)).content).toBe(shared); + expect(await readdir(textDirectory())).toHaveLength(2); + await restart(); + await forget("first"); + expect(await readdir(textDirectory())).toEqual([]); + await assertNoBytes(root, shared); + await restart(); + expect(await service.listSources(corpusId)).toEqual([]); + }); + + test.each([ + "blob-written", + "blob-verified", + "blob-published", + "manifest-prepared", + ])( + "%s interruption preserves the previous publication and collects unreferenced text on restart", + async (point) => { + await seed("first", "previous committed content"); + const before = await readFile(manifestFile(), "utf8"); + fault = point; + const job = await ingest( + "first", + "uncommitted replacement content", + ); + expect(job.state).toBe("failed"); + expect(job.error).toContain(point); + expect(await readFile(manifestFile(), "utf8")).toBe(before); + expect((await content("first")).content).toBe( + "previous committed content", + ); + await restart(); + expect(await readdir(textDirectory())).toHaveLength(1); + expect((await content("first")).content).toBe( + "previous committed content", + ); + await seed("first", "uncommitted replacement content"); + expect((await content("first")).content).toBe( + "uncommitted replacement content", + ); + }, + ); + + test.each(["purge-started", "purge-removed"])( + "source forgetting stays gated after %s and physically completes on restart", + async (point) => { + const secret = "uniquely forgotten source bytes"; + await seed("first", secret); + await seed("retained", "retained unrelated source"); + fault = point; + await expect(forget("first")).rejects.toThrow(point); + expect((await manifest()).authorizedSourceRemoval).toBeDefined(); + await expect(content("retained")).rejects.toThrow( + "pending purge recovery", + ); + await restart(); + expect((await manifest()).authorizedSourceRemoval).toBeUndefined(); + expect((await content("retained")).content).toBe( + "retained unrelated source", + ); + await assertNoBytes(root, secret); + }, + ); + + test.each(["purge-started", "purge-removed"])( + "corpus clear stays gated after %s and physically completes on restart", + async (point) => { + const secret = "uniquely cleared corpus bytes"; + await seed("first", secret); + fault = point; + await expect(service.clearCorpus(corpusId)).rejects.toThrow(point); + expect((await manifest()).pendingCorpusClear).toBeDefined(); + await expect(service.listSources(corpusId)).rejects.toThrow( + "pending purge recovery", + ); + await restart(); + expect((await manifest()).pendingCorpusClear).toBeUndefined(); + expect(await service.listSources(corpusId)).toEqual([]); + expect(await readdir(textDirectory())).toEqual([]); + await assertNoBytes(root, secret); + }, + ); + + test("successful clear purges all text and can accept fresh ingestion", async () => { + await seed("first", "discarded content"); + expect(await service.clearCorpus(corpusId)).toBe(1); + expect(await readdir(textDirectory())).toEqual([]); + await restart(); + await seed("new", "fresh content"); + expect((await content("new")).content).toBe("fresh content"); + }); + + test.each(["missing", "digest", "size", "hardlink"])( + "%s blob damage is explicit and never repaired from derived indexes", + async (damage) => { + await seed("first", "canonical text"); + await service.close(); + const stored = await manifest(); + const reference = stored.sources[0].revisions[0].text; + const file = path.join(textDirectory(), `${reference.sha256}.json`); + if (damage === "missing") await rm(file); + else if (damage === "digest") + await writeFile(file, '"other contents"'); + else if (damage === "size") { + reference.bytes++; + await writeFile(manifestFile(), JSON.stringify(stored)); + } else await link(file, path.join(root, "unexpected-hardlink")); + service = open(); + await expect(service.initialize()).rejects.toThrow( + damage === "missing" + ? "ENOENT" + : damage === "hardlink" + ? "Unsafe revision text blob" + : "Corrupt revision text blob", + ); + expect(await readFile(manifestFile(), "utf8")).toContain( + reference.sha256, + ); + }, + ); + + test.each([undefined, 0, 2])( + "rejects unsupported manifest storage version %s without deleting canonical data", + async (version) => { + await seed("first", "retained incompatible-store bytes"); + await service.close(); + const stored = { ...(await manifest()), storageVersion: version }; + const before = JSON.stringify(stored); + await writeFile(manifestFile(), before); + service = open(); + await expect(service.initialize()).rejects.toThrow( + /Unsupported.*reset\/reimport/, + ); + expect(await readFile(manifestFile(), "utf8")).toBe(before); + expect(await readdir(textDirectory())).toHaveLength(1); + }, + ); + + test("startup collects abandoned staged and published blobs without a second reference authority", async () => { + await seed("retained", "legitimately retained bytes"); + await service.close(); + const bytes = Buffer.from(JSON.stringify("abandoned process bytes")); + const hash = createHash("sha256").update(bytes).digest("hex"); + await writeFile(path.join(textDirectory(), `${hash}.json`), bytes); + await writeFile( + path.join( + textDirectory(), + `${hash}.00000000-0000-0000-0000-000000000000.tmp`, + ), + bytes, + ); + const abandonedManifest = `${manifestFile()}.00000000-0000-0000-0000-000000000000.staging`; + await writeFile( + abandonedManifest, + JSON.stringify({ title: "abandoned process bytes" }), + ); + service = open(); + await service.initialize(); + expect(await readdir(textDirectory())).toHaveLength(1); + await expect(stat(abandonedManifest)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect((await content("retained")).content).toBe( + "legitimately retained bytes", + ); + await assertNoBytes(root, "abandoned process bytes"); + }); + + test("text ownership is corpus-local even when two corpora retain identical bytes", async () => { + const shared = "shared across separate corpora"; + await seed("first", shared); + const other = await service.createCorpus("Other corpus"); + const accepted = await service.ingestDocument({ + corpusId: other.corpusId, + source: { + sourceId: "other", + sourceType: "text", + title: "other", + text: shared, + }, + }); + expect( + ( + await waitForMemoryJob(service, accepted.jobId, { + pollIntervalMs: 1, + }) + ).state, + ).toBe("complete"); + await forget("first"); + expect(await readdir(textDirectory())).toEqual([]); + await restart(); + expect( + ( + await service.getSourceContent({ + corpusId: other.corpusId, + sourceId: "other", + }) + ).content, + ).toBe(shared); + }); + + test("sharing does not overwrite an existing corrupt canonical blob", async () => { + await seed("first", "shared canonical text"); + const reference = (await manifest()).sources[0].revisions[0].text; + const file = path.join(textDirectory(), `${reference.sha256}.json`); + await writeFile(file, '"damaged bytes"'); + const job = await ingest("second", "shared canonical text"); + expect(job.state).toBe("failed"); + expect(job.error).toContain("Corrupt revision text blob"); + expect(await readFile(file, "utf8")).toBe('"damaged bytes"'); + expect((await manifest()).sources).toHaveLength(1); + }); + + test("an interrupted post-replacement purge cannot leave a cached reference to deleted bytes", async () => { + await seed("first", "old replaceable content"); + fault = "purge-removed"; + const job = await seed("first", "new replaceable content"); + expect(job.warnings.join(" ")).toContain("purge-removed"); + await seed("first", "old replaceable content"); + await restart(); + expect((await content("first")).content).toBe( + "old replaceable content", + ); + }); + + test("rejects traversal references before opening any referenced file", async () => { + const store = new RevisionTextStore(textDirectory()); + await expect( + store.read({ + sourceId: "first", + revisionId: "revision", + contentHash: "hash", + text: { sha256: "..\\manifest", bytes: 1 }, + }), + ).rejects.toThrow("Invalid canonical revision text reference"); + }); +}); diff --git a/ts/packages/memory/service/test/topicSnapshot.spec.ts b/ts/packages/memory/service/test/topicSnapshot.spec.ts index 56fae6a06d..df722ad246 100644 --- a/ts/packages/memory/service/test/topicSnapshot.spec.ts +++ b/ts/packages/memory/service/test/topicSnapshot.spec.ts @@ -15,6 +15,7 @@ import { import { createTopicSnapshot } from "../src/topicGeneration.js"; import { indexedTopicOccurrence } from "../src/topicIncidence.js"; import { TypedViewStore } from "../src/viewStore.js"; +import { ViewHistory } from "../src/viewHistory.js"; import type { TopicOccurrence, TopicSnapshot } from "../src/topicTypes.js"; import type { ViewBuildSnapshot } from "../src/viewTypes.js"; @@ -286,6 +287,83 @@ describe("complete scoped topic snapshots", () => { }); describe("topic persistence in existing view history", () => { + test("Topic reads reuse commit byte sizes across queries, publication and restart", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "topic-byte-sizes-")); + try { + const history = new ViewHistory( + path.join(root, "test", "views"), + () => ({ + views: {}, + topics: undefined as TopicSnapshot | undefined, + }), + ); + const state = { + views: {}, + topics: createTopicSnapshot( + input([occurrence("orders", "résumé review")]), + "offline", + ), + }; + const head = await history.commit( + null, + state, + {}, + "fixture", + "Initial", + ); + const expectedStateBytes = Buffer.byteLength(JSON.stringify(state)); + const expectedSnapshotBytes = Buffer.byteLength( + JSON.stringify(state.topics), + ); + for (const store of [ + new TypedViewStore(root), + new TypedViewStore(root), + ]) { + const stringify = import.meta.jest.spyOn(JSON, "stringify"); + try { + for (let i = 0; i < 2; i++) { + const page = await store.queryTopics({ + corpusId: "test", + }); + expect(page.io.stateBytes).toBe(expectedStateBytes); + expect(page.io.snapshotBytes).toBe( + expectedSnapshotBytes, + ); + } + expect( + stringify.mock.calls.filter( + ([value]) => + value && + typeof value === "object" && + "views" in value, + ), + ).toHaveLength(0); + expect( + stringify.mock.calls.filter( + ([value]) => + value && + typeof value === "object" && + "occurrences" in value, + ), + ).toHaveLength(1); + } finally { + stringify.mockRestore(); + } + } + const store = new TypedViewStore(root); + await store.queryTopics({ corpusId: "test" }); + const changed = { ...state, topics: undefined }; + await history.commit(head, changed, {}, "fixture", "Clear topics"); + const page = await store.queryTopics({ corpusId: "test" }); + expect(page.io.stateBytes).toBe( + Buffer.byteLength(JSON.stringify(changed)), + ); + expect(page.io.snapshotBytes).toBe(4); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + let directory: string; beforeEach(async () => { directory = await mkdtemp(path.join(os.tmpdir(), "memory-topics-")); diff --git a/ts/packages/memory/service/test/viewDrafts.spec.ts b/ts/packages/memory/service/test/viewDrafts.spec.ts index 6871f4c7fa..c22bb7cdd0 100644 --- a/ts/packages/memory/service/test/viewDrafts.spec.ts +++ b/ts/packages/memory/service/test/viewDrafts.spec.ts @@ -431,11 +431,26 @@ describe("typed draft views: service, history, privacy and CLI", () => { expect(await restarted.read()).toEqual({ head: initial, state: { value: "Original" }, + stateBytes: Buffer.byteLength( + JSON.stringify({ value: "Original" }), + ), }); expect(await restarted.history()).toHaveLength(1); }, ); + test("an absent history has zero committed bytes, including an undefined empty state", async () => { + const history = new ViewHistory( + path.join(root, "empty-history"), + () => undefined, + ); + expect(await history.read()).toEqual({ + head: null, + state: undefined, + stateBytes: 0, + }); + }); + test.each(["purge-prepared", "purge-swapped"] as ViewHistoryFaultPoint[])( "interrupted %s stays quarantined until restart completes full object database removal", async (point) => { diff --git a/ts/packages/memory/service/test/viewHistoryRecords.spec.ts b/ts/packages/memory/service/test/viewHistoryRecords.spec.ts new file mode 100644 index 0000000000..1ff9252928 --- /dev/null +++ b/ts/packages/memory/service/test/viewHistoryRecords.spec.ts @@ -0,0 +1,550 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import git from "isomorphic-git"; +import fs from "node:fs"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { inflateSync } from "node:zlib"; +import { ViewHistory } from "../src/viewHistory.js"; +import { TypedViewStore } from "../src/viewStore.js"; +import type { ViewHistoryEntry, ViewVersion } from "../src/viewTypes.js"; + +interface State { + views: Record; + corpusSequence: number; + unrelatedCorpusState: string; +} + +function version(viewId: string, revision: number): ViewVersion { + const createdAt = "2026-10-19T00:00:00.000Z"; + return { + corpusId: "corpus", + viewId, + revisionId: `${viewId}-${revision}`, + version: revision, + state: revision === 3 ? "archived" : "draft", + actor: "human-owner", + createdAt, + ...(revision > 1 + ? { baseRevisionId: `${viewId}-${revision - 1}` } + : {}), + definition: { + viewId, + revisionId: "definition-revision", + kind: "troubleshootingGuide", + selector: { + kind: "sources", + sources: [ + { sourceId: `${viewId}-source`, revisionId: "source-v1" }, + ], + }, + }, + content: { + kind: "troubleshootingGuide", + title: `${viewId} title ${revision}`, + citations: [], + sections: [ + { + id: "diagnose", + role: "diagnostic", + heading: "Diagnosis", + body: `${viewId} body ${revision}`, + }, + ], + }, + provenance: "merged", + relationships: [], + protectedTargets: ["title"], + edits: [ + { + id: "editorial-edit", + target: "title", + actor: "human-owner", + createdAt, + oldHash: "original-title-hash", + oldValue: "Original", + newValue: "Protected", + operation: "set", + status: "merged", + protection: ["title"], + unlockedTargets: ["sections.diagnose.body"], + }, + ], + }; +} + +function initialState(): State { + const target = version("target", 1); + target.generation = { + candidateId: "candidate", + content: structuredClone(target.content), + fingerprint: "generation-fingerprint", + input: { + corpusId: "corpus", + actor: "generator", + definition: target.definition, + expectedVersion: 0, + bounds: {}, + inputs: [], + influenceSources: [ + { sourceId: "target-source", revisionId: "source-v1" }, + ], + pipeline: "troubleshooting-v1", + model: "offline-model", + fingerprint: "generation-fingerprint", + }, + }; + const unrelated = version("unrelated", 1); + unrelated.content.sections[0].body = "unrelated-view-body".repeat(512); + return { + views: { target: [target], unrelated: [unrelated] }, + corpusSequence: 0, + unrelatedCorpusState: "unrelated-corpus-body".repeat(512), + }; +} + +function legacyEntries( + commits: Array<{ commitId: string; state: State }>, +): ViewHistoryEntry[] { + const seen = new Set(); + return commits + .reverse() + .flatMap(({ commitId, state }) => + (state.views.target ?? []).flatMap((version) => { + if (seen.has(version.revisionId)) return []; + seen.add(version.revisionId); + return [{ commitId, version }]; + }), + ) + .reverse(); +} + +function parsingCounts(calls: unknown[][]) { + const texts = calls.map((call) => String(call[0])); + return { + corpus: texts.filter((text) => text.includes('"corpusSequence":')) + .length, + unrelated: texts.filter((text) => text.includes("unrelated-view-body")) + .length, + selected: texts.filter( + (text) => + text.startsWith("[") && text.includes('"viewId":"target"'), + ).length, + }; +} + +describe("full-fidelity per-View history records", () => { + let root: string; + let directory: string; + beforeEach(async () => { + root = await mkdtemp(path.join(process.cwd(), ".b12-view-records-")); + directory = path.join(root, "corpus", "views"); + }); + afterEach(async () => { + import.meta.jest.restoreAllMocks(); + await rm(root, { recursive: true, force: true }); + }); + + function history(indexed = true) { + return new ViewHistory( + directory, + initialState, + undefined, + indexed ? (state) => state.views : undefined, + ); + } + + test.each([100, 1000])( + "%i commits match legacy ordering and exact records without parsing unrelated state", + async (count) => { + const indexed = history(); + const state = initialState(); + let head: string | null = null; + const started = performance.now(); + for (let index = 0; index < count; index++) { + state.corpusSequence = index; + state.views.unrelated[0].content.title = `Unrelated mutation ${index}`; + if (index === Math.floor(count / 3)) + state.views.target.push(version("target", 2)); + if (index === Math.floor(count / 2)) + state.views.target[0].actor = "later-metadata-mutation"; + if (index === Math.floor((2 * count) / 3)) + state.views.target.push(version("target", 3)); + if (index === count - 1) delete state.views.target; + head = await indexed.commit( + head, + state, + {}, + "test", + `Commit ${index}`, + ); + } + const writeMs = performance.now() - started; + const parse = import.meta.jest.spyOn(JSON, "parse"); + const baselineStart = performance.now(); + const expected = legacyEntries(await indexed.history()); + const baselineMs = performance.now() - baselineStart; + const baselineCounts = parsingCounts(parse.mock.calls); + expect(baselineCounts.corpus).toBe(count + 1); + parse.mockClear(); + + const store = new TypedViewStore(root); + const queryStart = performance.now(); + const actual = await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }); + const queryMs = performance.now() - queryStart; + expect(actual).toEqual(expected); + const counts = parsingCounts(parse.mock.calls); + expect(counts).toEqual({ corpus: 0, unrelated: 0, selected: 4 }); + expect(expected.map((entry) => entry.version.version)).toEqual([ + 3, 2, 1, + ]); + expect(expected.at(-1)?.version.actor).toBe("human-owner"); + parse.mockClear(); + expect( + await new TypedViewStore(root).getViewHistory({ + corpusId: "corpus", + viewId: "target", + revisionId: "target-2", + }), + ).toEqual( + expected.filter( + (entry) => entry.version.revisionId === "target-2", + ), + ); + expect(parsingCounts(parse.mock.calls).corpus).toBe(0); + parse.mockClear(); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "missing", + }), + ).toEqual([]); + expect(parsingCounts(parse.mock.calls)).toEqual({ + corpus: 0, + unrelated: 0, + selected: 0, + }); + process.stdout.write( + JSON.stringify({ + commits: count, + writeMs, + baselineMs, + queryMs, + baselineCorpusParses: baselineCounts.corpus, + baselineUnrelatedParses: baselineCounts.unrelated, + baselineSelectedRecordParses: baselineCounts.selected, + ...counts, + }) + "\n", + ); + }, + 180_000, + ); + + test("legacy indexes rebuild once, survive restart, and recover missing or malformed indexes", async () => { + const legacy = history(false); + const state = initialState(); + let head: string | null = null; + for (let index = 0; index < 3; index++) { + state.corpusSequence = index; + if (index) state.views.target.push(version("target", index + 1)); + head = await legacy.commit( + head, + state, + {}, + "test", + `Legacy ${index}`, + ); + } + const expected = legacyEntries(await legacy.history()); + const parse = import.meta.jest.spyOn(JSON, "parse"); + const store = new TypedViewStore(root); + const concurrent = await Promise.all([ + store.getViewHistory({ corpusId: "corpus", viewId: "target" }), + new TypedViewStore(root).getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ]); + expect(concurrent).toEqual([expected, expected]); + expect(parsingCounts(parse.mock.calls).corpus).toBe(3); + const indexDirectory = path.join( + directory, + "view-history.git", + "view-record-index", + ); + expect(await readdir(indexDirectory)).toHaveLength(3); + parse.mockClear(); + expect( + await new TypedViewStore(root).getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual(expected); + expect(parsingCounts(parse.mock.calls).corpus).toBe(0); + await writeFile(path.join(indexDirectory, head!), "interrupted-index"); + parse.mockClear(); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual(expected); + expect(parsingCounts(parse.mock.calls).corpus).toBe(1); + await writeFile(path.join(indexDirectory, head!), "a".repeat(40)); + parse.mockClear(); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual(expected); + expect(parsingCounts(parse.mock.calls).corpus).toBe(1); + await rm(indexDirectory, { recursive: true }); + parse.mockClear(); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual(expected); + expect(parsingCounts(parse.mock.calls).corpus).toBe(3); + parse.mockRestore(); + + state.corpusSequence++; + await history().commit(head, state, {}, "test", "Indexed successor"); + const parsed = import.meta.jest.spyOn(JSON, "parse"); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual(expected); + expect(parsingCounts(parsed.mock.calls).corpus).toBe(0); + }); + + test("head changes during indexed reads and scoped snapshots fail exact guards", async () => { + const indexed = history(); + const state = initialState(); + const head = await indexed.commit(null, state, {}, "test", "First"); + const original = git.readBlob; + let changed = false; + import.meta.jest + .spyOn(git, "readBlob") + .mockImplementation(async (options) => { + const blob = await original(options); + if (!changed) { + changed = true; + state.corpusSequence++; + await indexed.commit( + head, + state, + {}, + "test", + "Concurrent mutation", + ); + } + return blob; + }); + const store = new TypedViewStore(root); + await expect( + store.getViewHistory({ corpusId: "corpus", viewId: "target" }), + ).rejects.toThrow("View history changed during read operation"); + import.meta.jest.restoreAllMocks(); + await expect( + store.withReadSnapshot("corpus", async () => { + const current = await indexed.read(); + state.corpusSequence++; + await indexed.commit( + current.head, + state, + {}, + "test", + "Scoped mutation", + ); + return store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }); + }), + ).rejects.toThrow("View history changed during read operation"); + await expect( + indexed.commit(head, state, {}, "test", "Stale writer"), + ).rejects.toThrow("View history head conflict"); + }); + + test("unpublished record blobs never become a history commit", async () => { + const state = initialState(); + const indexed = history(); + const head = await indexed.commit(null, state, {}, "test", "Committed"); + const interrupted = new ViewHistory( + directory, + initialState, + async (point) => { + if (point === "ref") throw new Error("Interrupted publication"); + }, + (state) => state.views, + ); + state.views.target.push(version("target", 2)); + await expect( + interrupted.commit(head, state, {}, "test", "Uncommitted"), + ).rejects.toThrow("Interrupted publication"); + const entries = await new TypedViewStore(root).getViewHistory({ + corpusId: "corpus", + viewId: "target", + }); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + commitId: head, + version: { revisionId: "target-1" }, + }); + }); + + test.each(["purge-prepared", "purge-swapped", "purge-cleaned"] as const)( + "interrupted %s quarantines records until recovery removes old indexes and objects", + async (fault) => { + const state = initialState(); + const oldHead = await history(false).commit( + null, + state, + {}, + "test", + "Sensitive legacy state", + ); + const store = new TypedViewStore(root); + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }); + let failed = false; + const interrupted = new ViewHistory( + directory, + initialState, + async (point) => { + if (point === fault && !failed) { + failed = true; + throw new Error("Interrupted privacy purge"); + } + }, + (state) => state.views, + ); + const sanitize = (state: State) => { + delete state.views.target; + return state; + }; + const render = () => ({}); + await expect( + interrupted.purge("target-source", sanitize, render), + ).rejects.toThrow("Interrupted privacy purge"); + await expect( + store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).rejects.toThrow("quarantined pending privacy purge"); + await interrupted.recoverPurge(sanitize, render); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual([]); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "unrelated", + }), + ).toHaveLength(1); + expect(await readdir(directory)).toEqual(["view-history.git"]); + const gitdir = path.join(directory, "view-history.git"); + expect(await readdir(gitdir)).not.toContain("view-record-index"); + await expect( + git.readCommit({ + fs, + gitdir, + oid: oldHead, + }), + ).rejects.toThrow(); + }, + ); + + test.each(["source", "event", "clear"] as const)( + "%s forgetting removes legacy indexes, retained record blobs, and old generations", + async (kind) => { + const state = initialState(); + const secret = "private-historical-editorial-text"; + state.views.target[0].edits![0].newValue = secret; + if (kind === "event") + state.views.target[0].content.citations.push({ + sourceId: "event-source", + revisionId: "event-revision", + locator: "chars:0-7", + excerpt: "private", + evidence: { kind: "event", eventId: "private-event" }, + }); + const legacy = history(false); + const first = await legacy.commit( + null, + state, + {}, + "test", + "Legacy sensitive state", + ); + const store = new TypedViewStore(root); + await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }); + state.corpusSequence++; + const indexedHead = await history().commit( + first, + state, + {}, + "test", + "Indexed sensitive state", + ); + if (kind === "source") + await store.forgetSource("corpus", "target-source"); + else if (kind === "event") + await store.forgetEvent("corpus", "private-event"); + else await store.clearDerivedViews("corpus"); + expect( + await new TypedViewStore(root).getViewHistory({ + corpusId: "corpus", + viewId: "target", + }), + ).toEqual([]); + const retained = await store.getViewHistory({ + corpusId: "corpus", + viewId: "unrelated", + }); + expect(retained).toHaveLength(kind === "clear" ? 0 : 1); + expect(await readdir(directory)).toEqual(["view-history.git"]); + const gitdir = path.join(directory, "view-history.git"); + expect(await readdir(gitdir)).not.toContain("view-record-index"); + await expect( + git.readCommit({ fs, gitdir, oid: indexedHead }), + ).rejects.toThrow(); + const objects = path.join(gitdir, "objects"); + for (const prefix of await readdir(objects, { + withFileTypes: true, + })) { + if (!prefix.isDirectory() || !/^[a-f0-9]{2}$/.test(prefix.name)) + continue; + for (const file of await readdir( + path.join(objects, prefix.name), + )) { + const text = inflateSync( + await readFile(path.join(objects, prefix.name, file)), + ).toString("utf8"); + expect(text).not.toContain(secret); + expect(text).not.toContain("target-source"); + expect(text).not.toContain("private-event"); + } + } + }, + ); +}); diff --git a/ts/packages/memory/service/test/viewStoreSegments.spec.ts b/ts/packages/memory/service/test/viewStoreSegments.spec.ts new file mode 100644 index 0000000000..23baf175bb --- /dev/null +++ b/ts/packages/memory/service/test/viewStoreSegments.spec.ts @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { TypedViewStore } from "../src/viewStore.js"; +import { viewHash } from "../src/viewMerge.js"; +import type { ViewSaveRequest } from "../src/viewTypes.js"; + +test("View privacy cleanup preserves retained generation segments and removes forgotten index roots including segments", async () => { + const root = await mkdtemp(path.join(process.cwd(), ".b11-view-segments-")); + try { + const store = new TypedViewStore(root); + let head: string | null = null; + const indexRoot = path.join( + root, + "corpus", + "views", + "view-search-index", + ); + for (const viewId of ["forgotten", "retained"]) { + const request: ViewSaveRequest = { + corpusId: "corpus", + viewId, + expectedHead: head, + expectedVersion: 0, + definition: { + viewId, + kind: "troubleshootingGuide", + selector: { + kind: "sources", + sources: [ + { + sourceId: `${viewId}-source`, + revisionId: "source-revision", + }, + ], + }, + }, + content: { + kind: "troubleshootingGuide", + title: viewId, + citations: [], + sections: [ + { + id: "diagnose", + role: "diagnostic", + heading: "Diagnosis", + body: "Synthetic evidence.", + }, + ], + }, + relationships: [], + }; + head = (await store.saveViewDraft(request)).commitId; + const viewRoot = path.join(indexRoot, viewHash(viewId)); + for (const directory of [ + ".segments", + "generation-1", + "generation-2", + ]) { + const folder = path.join(viewRoot, directory); + await mkdir(folder, { recursive: true }); + await writeFile( + path.join(folder, "fixture.json"), + JSON.stringify({ viewId, directory }), + ); + } + } + + await store.forgetSource("corpus", "forgotten-source"); + expect(await readdir(indexRoot)).toEqual([viewHash("retained")]); + const retainedRoot = path.join(indexRoot, viewHash("retained")); + expect((await readdir(retainedRoot)).sort()).toEqual([ + ".segments", + "generation-1", + "generation-2", + ]); + for (const directory of [".segments", "generation-1", "generation-2"]) + expect( + JSON.parse( + await readFile( + path.join(retainedRoot, directory, "fixture.json"), + "utf8", + ), + ), + ).toEqual({ viewId: "retained", directory }); + + await store.clearDerivedViews("corpus"); + expect(await readdir(indexRoot)).toEqual([]); + await expect( + readdir(path.join(retainedRoot, ".segments")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/ts/tools/memory-maintenance-qualification/hot-path-benchmark.mjs b/ts/tools/memory-maintenance-qualification/hot-path-benchmark.mjs new file mode 100644 index 0000000000..efc5705dd3 --- /dev/null +++ b/ts/tools/memory-maintenance-qualification/hot-path-benchmark.mjs @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { performance } from "node:perf_hooks"; +import { FileMemoryService } from "../../packages/memory/service/dist/fileMemoryService.js"; +import { TypedViewStore } from "../../packages/memory/service/dist/viewStore.js"; +import { ViewHistory } from "../../packages/memory/service/dist/viewHistory.js"; +import { waitForMemoryJob } from "../../packages/memory/service/dist/rpcFacade.js"; +import { FakeSemanticCorpusIndex } from "../../packages/memory/service/dist/test/fakeSemanticCorpusIndex.js"; +import { compileRecipeProgram } from "../../packages/copilot-macros/dist/recipeCompiler.js"; +import { executeRecipeProgram } from "../../packages/copilot-macros/dist/recipeExecutor.js"; +import { + recipeDigest, + recipeSchemaDigest, +} from "../../packages/copilot-macros/dist/recipeCore.js"; +import { recipeArtifact } from "../../packages/copilot-macros/dist/test/recipeCatalogFixtures.js"; + +const productRoot = fileURLToPath(new URL("../../", import.meta.url)); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function deferred() { + let resolve; + const promise = new Promise((finish) => { + resolve = finish; + }); + return { promise, resolve }; +} + +function summary(samples) { + const sorted = [...samples].sort((a, b) => a - b); + const percentile = (p) => sorted[Math.ceil(p * sorted.length) - 1]; + return { samples, p50: percentile(0.5), p95: percentile(0.95) }; +} + +async function topicRead(root, bytes) { + const state = { views: {}, benchmarkPayload: "x".repeat(bytes) }; + const history = new ViewHistory(path.join(root, "corpus", "views"), () => ({ + views: {}, + })); + await history.commit( + null, + state, + {}, + "benchmark", + "Synthetic Topic read workload", + ); + const store = new TypedViewStore(root); + let fullStateSerializations = 0; + const stringify = JSON.stringify; + JSON.stringify = function (value, ...args) { + if (value?.benchmarkPayload) fullStateSerializations++; + return stringify(value, ...args); + }; + try { + const start = performance.now(); + const page = await store.queryTopics({ corpusId: "corpus" }); + assert.equal(page.io.stateBytes, Buffer.byteLength(stringify(state))); + return { + elapsedMs: performance.now() - start, + fullStateSerializations, + }; + } finally { + JSON.stringify = stringify; + } +} + +async function macroRun() { + const capabilities = Array.from({ length: 4 }, (_, i) => ({ + id: `read-${i}`, + transport: "native", + providerId: "benchmark", + operation: `read-${i}`, + contractVersion: "1", + inputSchema: { type: "null" }, + outputSchema: { type: "string" }, + schemaDigest: recipeSchemaDigest({ type: "null" }, { type: "string" }), + compatibility: "exact", + effect: "read", + })); + const source = `async function execute(api, params) { + let result = ""; + for (let i = 0; i < 4; i++) result += await api.tools.call("read-" + i, null); + return result; + }`; + const artifact = recipeArtifact("benchmark", "capture", { + source: { + language: "typescript", + text: source, + digest: recipeDigest(source), + }, + executable: compileRecipeProgram(source), + capabilities, + inputSchema: { type: "null" }, + }); + let inspections = 0, + active = 0, + maxActive = 0, + calls = 0; + const start = performance.now(); + const result = await executeRecipeProgram(artifact, null, { + runId: "benchmark", + binding: { + workspaceId: "benchmark", + profileId: "benchmark", + providerId: "benchmark", + accountId: "benchmark", + sessionId: "benchmark", + generation: "1", + }, + signal: new AbortController().signal, + host: { + inspectCapability: async (capability) => { + inspections++; + maxActive = Math.max(maxActive, ++active); + try { + await delay(5); + return { status: "available", capability }; + } finally { + active--; + } + }, + invokeCapability: async () => { + calls++; + return { status: "completed", value: "x" }; + }, + invokeSemantic: async () => { + throw new Error("Unexpected semantic call"); + }, + }, + }); + assert.equal(result.status, "completed", result.message); + assert.equal(result.output, "xxxx"); + assert.equal(calls, 4); + return { + elapsedMs: performance.now() - start, + inspections, + maxActive, + calls, + }; +} + +async function reindexRead(root) { + let gate; + const service = new FileMemoryService(root, { + viewDrafts: false, + informationalViews: false, + sourceSynopses: false, + indexFactory: (_id, directory) => + new (class extends FakeSemanticCorpusIndex { + async rebuild(documents) { + if (gate) { + gate.entered.resolve(); + await gate.release.promise; + } + return super.rebuild(documents); + } + })(directory), + }); + let reindex, read; + try { + const { corpusId } = await service.createCorpus("benchmark"); + const job = await service.ingestDocument({ + corpusId, + source: { + sourceId: "source", + sourceType: "text", + title: "Synthetic", + text: "Zephyr", + }, + }); + await waitForMemoryJob(service, job.jobId); + await service.search({ corpusId, query: "Zephyr" }); + gate = { entered: deferred(), release: deferred() }; + reindex = service.reindexCorpus(corpusId); + await Promise.race([ + gate.entered.promise, + reindex.then(() => { + throw new Error("Reindex skipped rebuild"); + }), + ]); + let finished = false; + const start = performance.now(); + read = service.search({ corpusId, query: "Zephyr" }).then((result) => { + finished = true; + assert.equal(result.matches.length, 1); + return performance.now() - start; + }); + await delay(30); + const readCompletedDuringPreparation = finished; + gate.release.resolve(); + const elapsedMs = await read; + await reindex; + return { elapsedMs, readCompletedDuringPreparation }; + } finally { + gate?.release.resolve(); + await Promise.allSettled([reindex, read].filter(Boolean)); + await service.close(); + } +} + +async function run(options) { + const root = await mkdtemp(path.join(os.tmpdir(), "typeagent-hot-path-")); + try { + const workloads = {}; + const cpu = process.cpuUsage(); + const start = performance.now(); + for (const [name, operation] of [ + ["topic-32k", (directory) => topicRead(directory, 32 * 1024)], + ["topic-256k", (directory) => topicRead(directory, 256 * 1024)], + ["macro-4-capabilities", () => macroRun()], + ["reindex-concurrent-read", (directory) => reindexRead(directory)], + ]) { + const samples = []; + for (let i = 0; i < options.samples + options.warmups; i++) { + const sample = await operation(path.join(root, `${name}-${i}`)); + if (i >= options.warmups) samples.push(sample); + } + workloads[name] = { + elapsedMs: summary(samples.map((s) => s.elapsedMs)), + observations: samples, + }; + } + const files = [ + "packages/memory/service/dist/fileMemoryService.js", + "packages/memory/service/dist/viewStore.js", + "packages/memory/service/dist/viewHistory.js", + "packages/copilot-macros/dist/recipeExecutor.js", + "packages/memory/service/dist/test/fakeSemanticCorpusIndex.js", + "packages/copilot-macros/dist/test/recipeCatalogFixtures.js", + ]; + const checkpoint = Object.fromEntries( + await Promise.all( + files.map(async (file) => [ + file, + createHash("sha256") + .update(await readFile(path.join(productRoot, file))) + .digest("hex"), + ]), + ), + ); + return { + schemaVersion: 1, + scope: "Offline synthetic first-milestone hot paths; not full scale or model quality qualification", + commit: execFileSync("git", ["rev-parse", "HEAD"], { + cwd: productRoot, + encoding: "utf8", + }).trim(), + dirty: + execFileSync("git", ["status", "--porcelain"], { + cwd: productRoot, + encoding: "utf8", + }).trim().length > 0, + node: process.version, + pnpm: execFileSync( + process.execPath, + ["-p", "require('./package.json').packageManager"], + { cwd: productRoot, encoding: "utf8" }, + ).trim(), + machine: { + platform: process.platform, + architecture: process.arch, + cpu: os.cpus()[0]?.model, + }, + configuration: { + ...options, + provider: "offline-stubs", + seed: "fixed", + inspectionDelayMs: 5, + preparationHoldMs: 30, + }, + checkpoint, + elapsedMs: performance.now() - start, + cpuMicros: process.cpuUsage(cpu), + peakRssBytes: process.resourceUsage().maxRSS * 1024, + workloads, + }; + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +const { values } = parseArgs({ + options: { + samples: { type: "string", default: "5" }, + warmups: { type: "string", default: "1" }, + output: { type: "string" }, + }, +}); +const samples = Number(values.samples), + warmups = Number(values.warmups); +assert.ok(Number.isSafeInteger(samples) && samples > 0 && samples <= 100); +assert.ok(Number.isSafeInteger(warmups) && warmups >= 0 && warmups <= 10); +const result = JSON.stringify(await run({ samples, warmups }), null, 2) + "\n"; +if (values.output) + await writeFile(path.resolve(values.output), result, { flag: "wx" }); +else process.stdout.write(result); diff --git a/ts/tools/memory-maintenance-qualification/macro-runtime-benchmark.mjs b/ts/tools/memory-maintenance-qualification/macro-runtime-benchmark.mjs new file mode 100644 index 0000000000..533fac40f0 --- /dev/null +++ b/ts/tools/memory-maintenance-qualification/macro-runtime-benchmark.mjs @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; +import * as runtime from "../../packages/copilot-macros/dist/recipeExecutor.js"; +import { compileRecipeProgram } from "../../packages/copilot-macros/dist/recipeCompiler.js"; +import { + recipeDigest, + recipeSchemaDigest, +} from "../../packages/copilot-macros/dist/recipeCore.js"; +import { recipeArtifact } from "../../packages/copilot-macros/dist/test/recipeCatalogFixtures.js"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +const binding = { + workspaceId: "benchmark", + profileId: "benchmark", + providerId: "benchmark", + accountId: "benchmark", + sessionId: "benchmark", + generation: "1", +}; +const capability = { + id: "read", + transport: "native", + providerId: "benchmark", + operation: "read", + contractVersion: "1", + inputSchema: { type: "null" }, + outputSchema: { type: "string" }, + schemaDigest: recipeSchemaDigest({ type: "null" }, { type: "string" }), + compatibility: "exact", + effect: "read", +}; + +function artifact(parallel) { + const source = parallel + ? `async function execute(api, params) { + return (await Promise.all(Array.from({length: 8}, () => api.tools.call("read", null)))).join(""); + }` + : `async function execute(api, params) { return await api.tools.call("read", null); }`; + return recipeArtifact("runtime-benchmark", "capture", { + source: { + language: "typescript", + text: source, + digest: recipeDigest(source), + }, + executable: compileRecipeProgram(source), + capabilities: [capability], + inputSchema: { type: "null" }, + }); +} + +function distribution(samples) { + const sorted = [...samples].sort((a, b) => a - b); + return { + samples, + p50: sorted[Math.ceil(sorted.length * 0.5) - 1], + p95: sorted[Math.ceil(sorted.length * 0.95) - 1], + }; +} + +async function workload(name, width, parallel, samples) { + const owner = runtime.RecipeExecutor + ? new runtime.RecipeExecutor() + : { execute: runtime.executeRecipeProgram, close: async () => {} }; + const program = artifact(parallel); + const observations = []; + let active = 0, + maxActive = 0; + try { + for (let round = 0; round <= samples; round++) { + maxActive = 0; + const cpu = process.cpuUsage(); + const start = performance.now(); + const results = await Promise.all( + Array.from({ length: width }, (_, index) => + owner.execute( + program, + null, + { + runId: `${name}-${round}-${index}`, + binding, + signal: new AbortController().signal, + host: { + inspectCapability: async (capability) => ({ + status: "available", + capability, + }), + invokeCapability: async () => { + maxActive = Math.max(maxActive, ++active); + try { + await new Promise((resolve) => + setTimeout(resolve, 10), + ); + return { + status: "completed", + value: "x", + }; + } finally { + active--; + } + }, + invokeSemantic: async () => { + throw new Error("Unexpected semantic call"); + }, + }, + }, + { timeoutMs: 30_000 }, + ), + ), + ); + for (const result of results) { + assert.equal(result.status, "completed", result.message); + assert.equal(result.output, parallel ? "xxxxxxxx" : "x"); + assert.equal(result.completedCallIds.length, parallel ? 8 : 1); + } + observations.push({ + phase: round === 0 ? "cold" : "warm", + elapsedMs: performance.now() - start, + maxActiveCalls: maxActive, + queueMs: results.map( + (result) => result.performance?.workerQueueMs ?? null, + ), + workerReused: results.map( + (result) => result.performance?.workerReused ?? null, + ), + rssBytes: process.memoryUsage().rss, + cpuMicros: process.cpuUsage(cpu), + }); + } + return { + width, + parallel, + observations, + warmElapsedMs: distribution( + observations.slice(1).map((value) => value.elapsedMs), + ), + }; + } finally { + await owner.close(); + } +} + +export async function runMacroRuntimeBenchmark(samples) { + const workloads = {}; + for (const [name, width, parallel] of [ + ["single", 1, false], + ["parallel-reads", 1, true], + ["concurrent-4", 4, false], + ["concurrent-8", 8, false], + ]) + workloads[name] = await workload(name, width, parallel, samples); + const checkpoint = {}; + for (const file of [ + "recipeExecutor.js", + "recipeExecutorWorker.js", + "recipeWorkerPool.js", + ]) { + try { + checkpoint[file] = createHash("sha256") + .update( + await readFile( + path.join( + root, + "packages", + "copilot-macros", + "dist", + file, + ), + ), + ) + .digest("hex"); + } catch (error) { + if (file !== "recipeWorkerPool.js" || error.code !== "ENOENT") + throw error; + checkpoint[file] = null; + } + } + return { + schemaVersion: 1, + scope: "Offline synthetic B4-B5; no live quality or scale qualification", + mode: runtime.RecipeExecutor + ? "owned-reusable-workers" + : "one-worker-per-run", + commit: execFileSync("git", ["rev-parse", "HEAD"], { + cwd: root, + encoding: "utf8", + }).trim(), + dirty: + execFileSync("git", ["status", "--porcelain"], { + cwd: root, + encoding: "utf8", + }).trim().length > 0, + checkpoint, + node: process.version, + machine: { + platform: process.platform, + architecture: process.arch, + cpu: os.cpus()[0]?.model, + }, + configuration: { + samples, + coldRounds: 1, + provider: "offline-stubs", + callDelayMs: 10, + seed: "fixed", + }, + peakRssBytes: process.resourceUsage().maxRSS * 1024, + workloads, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { + samples: { type: "string", default: "5" }, + output: { type: "string" }, + }, + }); + const samples = Number(values.samples); + if (!Number.isSafeInteger(samples) || samples < 1 || samples > 100) + throw new Error("samples must be between 1 and 100"); + const result = + JSON.stringify(await runMacroRuntimeBenchmark(samples), null, 2) + "\n"; + if (values.output) await writeFile(values.output, result, { flag: "wx" }); + else process.stdout.write(result); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href +) { + await main(); +} diff --git a/ts/tools/memory-maintenance-qualification/storage-write-benchmark.mjs b/ts/tools/memory-maintenance-qualification/storage-write-benchmark.mjs new file mode 100644 index 0000000000..9d278aaf9e --- /dev/null +++ b/ts/tools/memory-maintenance-qualification/storage-write-benchmark.mjs @@ -0,0 +1,547 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import fs from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { FileMemoryService } from "../../packages/memory/service/dist/fileMemoryService.js"; +import { createKnowProCorpusIndex } from "../../packages/memory/service/dist/knowProCorpusIndex.js"; +import { createDocMemorySettings } from "../../packages/memory/conversation/dist/docMemory.js"; +import { + writeSegmentedJson, + readSegmentedJson, +} from "../../packages/knowPro/dist/segmentedJson.js"; +import { waitForMemoryJob } from "../../packages/memory/service/dist/rpcFacade.js"; +import { FakeSemanticCorpusIndex } from "../../packages/memory/service/dist/test/fakeSemanticCorpusIndex.js"; + +const productRoot = fileURLToPath(new URL("../../", import.meta.url)); + +function summary(samples) { + const ordered = [...samples].sort((a, b) => a - b); + return { + samples, + p50: ordered[Math.ceil(ordered.length * 0.5) - 1], + p95: ordered[Math.ceil(ordered.length * 0.95) - 1], + }; +} + +async function workload(root, count, samples) { + const counters = { + manifestWrites: 0, + manifestBytes: 0, + textBlobWrites: 0, + textBlobBytes: 0, + providerDocuments: 0, + }; + let measuring = false; + class CountingIndex extends FakeSemanticCorpusIndex { + async rebuild(documents) { + if (measuring) counters.providerDocuments += documents.length; + await super.rebuild(documents); + } + } + const service = new FileMemoryService(root, { + viewDrafts: false, + informationalViews: false, + sourceSynopses: false, + indexFactory: (_, directory) => new CountingIndex(directory), + }); + const writeFile = fs.writeFile; + fs.writeFile = async function (file, data, ...args) { + const result = await writeFile.call(this, file, data, ...args); + if (measuring && typeof file === "string") { + const bytes = + typeof data === "string" + ? Buffer.byteLength(data) + : data.byteLength; + if (path.basename(file).startsWith("manifest.json.")) { + counters.manifestWrites++; + counters.manifestBytes += bytes; + } else if (file.includes(`${path.sep}revision-text${path.sep}`)) { + counters.textBlobWrites++; + counters.textBlobBytes += bytes; + } + } + return result; + }; + syncBuiltinESMExports(); + try { + const corpus = await service.createCorpus("Fixed storage workload"); + const corpusId = corpus.corpusId; + const payload = "offline retained text ".repeat(3200).slice(0, 65536); + async function ingest(sourceId, suffix) { + const accepted = await service.ingestDocument({ + corpusId, + source: { + sourceId, + sourceType: "text", + title: sourceId, + text: `${payload}${sourceId}:${suffix}`, + }, + pipeline: { updatePolicy: "replaceActiveRevision" }, + }); + const job = await waitForMemoryJob(service, accepted.jobId); + assert.equal(job.state, "complete", job.error); + } + for (let i = 0; i < count; i++) await ingest(`source-${i}`, "seed"); + const observations = []; + for (let i = 0; i < samples; i++) { + for (const key of Object.keys(counters)) counters[key] = 0; + measuring = true; + const start = performance.now(); + await ingest("source-0", `replacement-${i}`); + const elapsedMs = performance.now() - start; + measuring = false; + const retained = await service.getSourceContent({ + corpusId, + sourceId: `source-${count - 1}`, + offset: payload.length, + }); + assert.ok(retained.content.includes(`source-${count - 1}:seed`)); + const manifestBytes = ( + await fs.stat(path.join(root, corpusId, "manifest.json")) + ).size; + observations.push({ + elapsedMs, + ...counters, + manifestSizeBytes: manifestBytes, + }); + } + return { + sourceCount: count, + textBytesPerSource: 65536, + elapsedMs: summary(observations.map((sample) => sample.elapsedMs)), + observations, + }; + } finally { + measuring = false; + fs.writeFile = writeFile; + syncBuiltinESMExports(); + await service.close(); + } +} + +async function batchWorkload(root, samples) { + const observations = []; + for (let sample = 0; sample < samples; sample++) { + const counters = { + indexCopies: 0, + manifestWrites: 0, + manifestBytes: 0, + providerBatches: 0, + providerDocuments: 0, + }; + let measuring = false; + class CountingIndex extends FakeSemanticCorpusIndex { + async append(documents) { + if (measuring) { + counters.providerBatches++; + counters.providerDocuments += documents.length; + } + await super.append(documents); + } + } + const directory = path.join(root, `batch-${sample}`); + const service = new FileMemoryService(directory, { + viewDrafts: false, + informationalViews: false, + sourceSynopses: false, + indexFactory: (_, indexDirectory) => + new CountingIndex(indexDirectory), + }); + const writeFile = fs.writeFile, + cp = fs.cp; + fs.writeFile = async function (file, data, ...args) { + const result = await writeFile.call(this, file, data, ...args); + if ( + measuring && + typeof file === "string" && + path.basename(file).startsWith("manifest.json.") + ) { + counters.manifestWrites++; + counters.manifestBytes += Buffer.byteLength(data); + } + return result; + }; + fs.cp = async function (source, destination, ...args) { + const result = await cp.call(this, source, destination, ...args); + if (measuring) counters.indexCopies++; + return result; + }; + syncBuiltinESMExports(); + try { + const { corpusId } = await service.createCorpus( + "Batch storage workload", + ); + const seed = await service.ingestDocument({ + corpusId, + source: { + sourceId: "seed", + sourceType: "text", + title: "seed", + text: "Retained seed content", + }, + }); + assert.equal( + ( + await waitForMemoryJob(service, seed.jobId, { + pollIntervalMs: 5, + }) + ).state, + "complete", + ); + measuring = true; + const start = performance.now(); + let batch = await service.startBatchImport({ + corpusId, + idempotencyKey: "fixed-batch", + documents: Array.from({ length: 8 }, (_, i) => ({ + source: { + sourceId: `member-${i}`, + sourceType: "text", + title: `member-${i}`, + text: `${"offline acquired text ".repeat(500)}unique member ${i}`, + }, + })), + }); + while (batch.state === "running") { + await new Promise((resolve) => setTimeout(resolve, 5)); + batch = await service.getBatchImport(batch.batchId); + } + const elapsedMs = performance.now() - start; + measuring = false; + assert.equal(batch.state, "complete", JSON.stringify(batch)); + assert.equal(batch.members.length, 8); + assert.ok( + batch.members.every( + (member) => member.state === "complete" && member.jobId, + ), + ); + assert.equal((await service.listSources(corpusId)).length, 9); + for (let i = 0; i < 8; i++) + assert.ok( + ( + await service.getSourceContent({ + corpusId, + sourceId: `member-${i}`, + offset: 11000, + }) + ).content.includes(`unique member ${i}`), + ); + observations.push({ elapsedMs, ...counters }); + } finally { + measuring = false; + fs.writeFile = writeFile; + fs.cp = cp; + syncBuiltinESMExports(); + await service.close(); + } + } + return { + memberCount: 8, + retainedSourceCount: 1, + observations, + elapsedMs: summary(observations.map((sample) => sample.elapsedMs)), + }; +} + +async function deltaWorkload(root, samples, delta) { + const previousProvider = process.env.TYPEAGENT_EMBEDDING_PROVIDER; + process.env.TYPEAGENT_EMBEDDING_PROVIDER = "none"; + let providerDocuments = 0; + const languageModel = { + completionSettings: {}, + complete: async () => ({ + success: true, + data: JSON.stringify({ + searchExpressions: [ + { + rewrittenQuery: "Zephyr", + filters: [ + { + entitySearchTerms: [ + { name: "Zephyr", isNamePronoun: false }, + ], + }, + ], + }, + ], + }), + }), + }; + const service = new FileMemoryService(root, { + viewDrafts: false, + informationalViews: false, + sourceSynopses: false, + indexFactory: (corpusId, directory) => { + const index = createKnowProCorpusIndex(corpusId, directory, () => { + const settings = createDocMemorySettings( + 64, + undefined, + languageModel, + ); + settings.embeddingSize = 0; + const knowledge = () => ({ + entities: [{ name: "Zephyr", type: ["service"] }], + actions: [], + inverseActions: [], + topics: [], + }); + settings.conversationSettings.semanticRefIndexSettings.knowledgeExtractor = + { + settings: { maxContextLength: 1000 }, + extract: async () => knowledge(), + extractWithRetry: async () => { + providerDocuments++; + return { success: true, data: knowledge() }; + }, + }; + return settings; + }); + if (!delta) index.replace = undefined; + return index; + }, + }); + try { + const { corpusId } = await service.createCorpus( + "Fixed real KnowPro delta workload", + ); + const ingest = async (sourceId, suffix) => { + const accepted = await service.ingestDocument({ + corpusId, + source: { + sourceId, + sourceType: "text", + title: sourceId, + text: `Zephyr ${"Retained knowledge. ".repeat(180)}${sourceId}:${suffix}`, + }, + pipeline: { updatePolicy: "retainRevisionHistory" }, + }); + const job = await waitForMemoryJob(service, accepted.jobId); + assert.equal(job.state, "complete", job.error); + }; + for (let i = 0; i < 8; i++) await ingest(`source-${i}`, "seed"); + const observations = []; + for (let i = 0; i < samples; i++) { + providerDocuments = 0; + const started = performance.now(); + await ingest("source-0", `replacement-${i}`); + const elapsedMs = performance.now() - started; + assert.equal(providerDocuments, delta ? 1 : 8); + const hits = await service.search({ + corpusId, + query: "Zephyr", + limit: 20, + }); + assert.deepEqual( + hits.matches.map((hit) => hit.sourceId).sort(), + Array.from({ length: 8 }, (_, n) => `source-${n}`), + ); + assert.equal( + hits.matches + .find((hit) => hit.sourceId === "source-0") + ?.snippet.includes(`replacement-${i}`), + true, + ); + observations.push({ + elapsedMs, + providerDocuments, + groundedSourceMatches: hits.matches.length, + }); + } + return { + sourceCount: 8, + delta, + elapsedMs: summary(observations.map((value) => value.elapsedMs)), + observations, + }; + } finally { + await service.close(); + if (previousProvider === undefined) + delete process.env.TYPEAGENT_EMBEDDING_PROVIDER; + else process.env.TYPEAGENT_EMBEDDING_PROVIDER = previousProvider; + } +} + +async function segmentWorkload(root, samples) { + const data = { + records: Array.from({ length: 1024 }, (_, ordinal) => ({ + ordinal, + text: `${"Retained source knowledge. ".repeat(24)}${ordinal}`, + })), + vectors: [[0.6, 0.8]], + }; + const initial = path.join(root, "initial", "corpus_data.json"); + await writeSegmentedJson(initial, data, { messages: 1024 }); + const segments = path.join(root, ".segments"); + const observations = []; + for (let i = 0; i < samples; i++) { + const retained = new Map( + await Promise.all( + (await fs.readdir(segments)).map(async (name) => [ + name, + await fs.stat(path.join(segments, name)), + ]), + ), + ); + data.records.push({ ordinal: 1024 + i, text: `Changed source ${i}` }); + const file = path.join(root, `generation-${i}`, "corpus_data.json"); + await writeSegmentedJson(file, data, { messages: data.records.length }); + let segmentBytes = 0; + let segmentWrites = 0; + for (const name of await fs.readdir(segments)) { + const info = await fs.stat(path.join(segments, name)); + if (retained.has(name)) { + assert.equal(info.ino, retained.get(name).ino); + assert.equal(info.mtimeMs, retained.get(name).mtimeMs); + } else { + segmentWrites++; + segmentBytes += info.size; + } + } + const descriptorBytes = (await fs.stat(file)).size; + const baseline = JSON.stringify(data); + await fs.writeFile( + path.join(root, "reconstructed-monolithic.json"), + baseline, + ); + assert.deepEqual( + await readSegmentedJson( + file, + JSON.parse(await fs.readFile(file, "utf8")), + ), + data, + ); + assert.ok( + segmentBytes + descriptorBytes < Buffer.byteLength(baseline) / 20, + ); + observations.push({ + monolithicJsonBytes: Buffer.byteLength(baseline), + segmentBytes, + segmentWrites, + descriptorBytes, + retainedSegmentsRewritten: 0, + }); + } + return { + scope: "Identical synthetic JSON values, reconstructed prior whole-JSON serialization versus actual immutable node and descriptor bytes. Excludes index metadata, GC reads/deletes, fsyncs and device I/O; not a full live index benchmark.", + recordCount: 1024, + observations, + }; +} + +export async function runStorageWriteBenchmark(samples, kind = "replace") { + const root = await fs.mkdtemp( + path.join(os.tmpdir(), "typeagent-storage-write-"), + ); + const cpu = process.cpuUsage(); + try { + const workloads = {}; + if (kind === "delta") { + workloads["replacement-full-rebuild"] = await deltaWorkload( + path.join(root, "rebuild"), + samples, + false, + ); + workloads["replacement-delta"] = await deltaWorkload( + path.join(root, "delta"), + samples, + true, + ); + workloads["segmented-append-1024"] = await segmentWorkload( + path.join(root, "segmented"), + samples, + ); + } else if (kind === "batch") + workloads["batch-eight-new-sources"] = await batchWorkload( + root, + samples, + ); + else + for (const count of [8, 64]) + workloads[`replace-one-of-${count}`] = await workload( + path.join(root, `sources-${count}`), + count, + samples, + ); + const files = [ + "packages/memory/service/dist/fileMemoryService.js", + "packages/memory/service/dist/revisionTextStore.js", + "packages/memory/service/dist/batchImport.js", + "packages/memory/service/dist/boundedIngestionGroup.js", + "packages/memory/service/dist/test/fakeSemanticCorpusIndex.js", + "packages/memory/service/dist/knowProCorpusIndex.js", + "packages/knowPro/dist/segmentedJson.js", + "packages/knowPro/dist/search.js", + "tools/memory-maintenance-qualification/storage-write-benchmark.mjs", + ]; + const checkpoint = {}; + for (const file of files) + checkpoint[file] = createHash("sha256") + .update(await fs.readFile(path.join(productRoot, file))) + .digest("hex"); + return { + schemaVersion: 1, + scope: "Offline storage write amplification. replace/batch use a fake index; delta uses real KnowPro with deterministic extraction and no embeddings, comparing the same candidate with replace capability disabled/enabled. Counts JSON payloads, not physical device I/O. Segment observations reconstruct prior whole-JSON serialization on identical values. No live latency/RSS certification.", + commit: execFileSync("git", ["rev-parse", "HEAD"], { + cwd: productRoot, + encoding: "utf8", + }).trim(), + dirty: + execFileSync("git", ["status", "--porcelain"], { + cwd: productRoot, + encoding: "utf8", + }).trim().length > 0, + checkpoint, + node: process.version, + machine: { + platform: process.platform, + architecture: process.arch, + cpu: os.cpus()[0]?.model, + }, + configuration: { + samples, + workload: kind, + provider: "offline-stubs", + seed: "fixed", + }, + cpuMicros: process.cpuUsage(cpu), + peakRssBytes: process.resourceUsage().maxRSS * 1024, + workloads, + }; + } finally { + await fs.rm(root, { recursive: true, force: true }); + } +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const { values } = parseArgs({ + options: { + samples: { type: "string", default: "5" }, + workload: { type: "string", default: "replace" }, + output: { type: "string" }, + }, + }); + const samples = Number(values.samples); + assert.ok(Number.isSafeInteger(samples) && samples > 0 && samples <= 100); + assert.ok(["replace", "batch", "delta"].includes(values.workload)); + const result = + JSON.stringify( + await runStorageWriteBenchmark(samples, values.workload), + null, + 2, + ) + "\n"; + if (values.output) + await fs.writeFile(path.resolve(values.output), result, { flag: "wx" }); + else process.stdout.write(result); +} diff --git a/ts/tools/scripts/test/macroRuntimeBenchmark.spec.mjs b/ts/tools/scripts/test/macroRuntimeBenchmark.spec.mjs new file mode 100644 index 0000000000..cc62ace483 --- /dev/null +++ b/ts/tools/scripts/test/macroRuntimeBenchmark.spec.mjs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +test( + "macro runtime runner verifies warm reuse and bounded read concurrency", + { timeout: 60_000 }, + async () => { + const runner = fileURLToPath( + new URL( + "../../memory-maintenance-qualification/macro-runtime-benchmark.mjs", + import.meta.url, + ), + ); + const { stdout } = await promisify(execFile)( + process.execPath, + [runner, "--samples", "1"], + { timeout: 55_000 }, + ); + const result = JSON.parse(stdout); + assert.equal(result.schemaVersion, 1); + assert.equal(result.mode, "owned-reusable-workers"); + assert.equal(result.configuration.provider, "offline-stubs"); + for (const hash of Object.values(result.checkpoint)) + assert.match(hash, /^[a-f0-9]{64}$/); + assert.deepEqual(Object.keys(result.workloads), [ + "single", + "parallel-reads", + "concurrent-4", + "concurrent-8", + ]); + for (const workload of Object.values(result.workloads)) { + assert.equal(workload.observations.length, 2); + assert.equal(workload.observations[0].phase, "cold"); + const warm = workload.observations[1]; + assert.equal(warm.phase, "warm"); + assert.ok(warm.workerReused.every((value) => value === true)); + assert.ok( + warm.queueMs.every( + (value) => Number.isFinite(value) && value >= 0, + ), + ); + assert.equal(workload.warmElapsedMs.samples.length, 1); + assert.ok( + warm.maxActiveCalls <= (workload.parallel ? 4 : workload.width), + ); + } + assert.equal( + result.workloads["parallel-reads"].observations[1].maxActiveCalls, + 4, + ); + }, +); diff --git a/ts/tools/scripts/test/memoryHotPathBenchmark.spec.mjs b/ts/tools/scripts/test/memoryHotPathBenchmark.spec.mjs new file mode 100644 index 0000000000..c1edfc17be --- /dev/null +++ b/ts/tools/scripts/test/memoryHotPathBenchmark.spec.mjs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const execute = promisify(execFile); +const runner = fileURLToPath( + new URL( + "../../memory-maintenance-qualification/hot-path-benchmark.mjs", + import.meta.url, + ), +); + +test( + "offline hot-path runner emits checked, reproducible observations", + { timeout: 60_000 }, + async () => { + const { stdout } = await execute( + process.execPath, + [runner, "--samples", "1", "--warmups", "0"], + { timeout: 55_000 }, + ); + const result = JSON.parse(stdout); + assert.equal(result.schemaVersion, 1); + assert.equal(result.configuration.provider, "offline-stubs"); + assert.match(result.commit, /^[0-9a-f]{40}$/); + for (const hash of Object.values(result.checkpoint)) + assert.match(hash, /^[0-9a-f]{64}$/); + for (const workload of Object.values(result.workloads)) { + assert.equal(workload.observations.length, 1); + assert.ok(workload.elapsedMs.p50 >= 0); + assert.ok(workload.elapsedMs.p95 >= workload.elapsedMs.p50); + } + assert.equal( + result.workloads["topic-256k"].observations[0] + .fullStateSerializations, + 0, + ); + assert.equal( + result.workloads["macro-4-capabilities"].observations[0].maxActive, + 4, + ); + assert.equal( + typeof result.workloads["reindex-concurrent-read"].observations[0] + .readCompletedDuringPreparation, + "boolean", + ); + }, +); + +test("offline hot-path runner rejects invalid sample bounds", async () => { + await assert.rejects( + execute(process.execPath, [runner, "--samples", "0"]), + /Command failed/, + ); +}); diff --git a/ts/tools/scripts/test/storageWriteBenchmark.spec.mjs b/ts/tools/scripts/test/storageWriteBenchmark.spec.mjs new file mode 100644 index 0000000000..7e824010d3 --- /dev/null +++ b/ts/tools/scripts/test/storageWriteBenchmark.spec.mjs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const execute = promisify(execFile); +const runner = fileURLToPath( + new URL( + "../../memory-maintenance-qualification/storage-write-benchmark.mjs", + import.meta.url, + ), +); + +test( + "storage runner verifies bounded changed-text writes, not a latency proxy", + { timeout: 90_000 }, + async () => { + const { stdout } = await execute( + process.execPath, + [runner, "--samples", "1"], + { timeout: 85_000 }, + ); + const result = JSON.parse(stdout); + assert.equal(result.configuration.provider, "offline-stubs"); + assert.match(result.commit, /^[a-f0-9]{40}$/); + for (const hash of Object.values(result.checkpoint)) + assert.match(hash, /^[a-f0-9]{64}$/); + for (const workload of Object.values(result.workloads)) { + assert.equal(workload.observations.length, 1); + const sample = workload.observations[0]; + assert.equal(sample.manifestWrites, 1); + assert.equal(sample.textBlobWrites, 1); + assert.ok( + sample.textBlobBytes >= 65536 && sample.textBlobBytes < 66000, + ); + assert.ok(sample.manifestBytes < workload.sourceCount * 2000); + assert.equal(sample.providerDocuments, workload.sourceCount); + } + }, +); + +test( + "batch runner verifies fewer copies/publications without skipping member provider work", + { timeout: 60_000 }, + async () => { + const { stdout } = await execute( + process.execPath, + [runner, "--workload", "batch", "--samples", "1"], + { timeout: 55_000 }, + ); + const report = JSON.parse(stdout); + const sample = + report.workloads["batch-eight-new-sources"].observations[0]; + assert.equal(sample.providerDocuments, 8); + assert.ok(sample.indexCopies >= 4 && sample.indexCopies < 8); + assert.equal(sample.manifestWrites, sample.indexCopies); + assert.equal(sample.providerBatches, sample.indexCopies); + }, +); + +test("storage runner rejects invalid sample bounds", async () => { + await assert.rejects(execute(process.execPath, [runner, "--samples", "0"])); +}); + +test( + "delta runner verifies real extraction work and bounded segmented writes", + { timeout: 90_000 }, + async () => { + const { stdout } = await execute( + process.execPath, + [runner, "--workload", "delta", "--samples", "1"], + { timeout: 85_000 }, + ); + const report = JSON.parse(stdout); + assert.equal( + report.workloads["replacement-full-rebuild"].observations[0] + .providerDocuments, + 8, + ); + assert.equal( + report.workloads["replacement-delta"].observations[0] + .providerDocuments, + 1, + ); + const sample = + report.workloads["segmented-append-1024"].observations[0]; + assert.ok( + sample.segmentBytes + sample.descriptorBytes < + sample.monolithicJsonBytes / 20, + ); + assert.equal(sample.retainedSegmentsRewritten, 0); + }, +); From 223cc68aab7cc362926e0347ca3e816f719f88f1 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Mon, 19 Oct 2026 22:48:41 +0000 Subject: [PATCH 03/10] style: apply prettier formatting and policy fixes --- ts/packages/agent-flows/test/scriptExecutor.spec.ts | 3 +++ ts/packages/copilot-macros/test/macroGrammarCache.spec.ts | 3 +++ ts/packages/knowPro/test/embeddingFilter.spec.ts | 3 +++ ts/packages/memory/service/test/viewHistoryEntries.spec.ts | 3 +++ 4 files changed, 12 insertions(+) diff --git a/ts/packages/agent-flows/test/scriptExecutor.spec.ts b/ts/packages/agent-flows/test/scriptExecutor.spec.ts index ec6c165c41..a228146b8b 100644 --- a/ts/packages/agent-flows/test/scriptExecutor.spec.ts +++ b/ts/packages/agent-flows/test/scriptExecutor.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { jest } from "@jest/globals"; import { createScriptExecutor } from "../src/execution/scriptExecutor.js"; diff --git a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts index 8d918eee56..3e2c771da4 100644 --- a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts +++ b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import type { CopilotToolMacro } from "../src/contracts.js"; const jest = ( diff --git a/ts/packages/knowPro/test/embeddingFilter.spec.ts b/ts/packages/knowPro/test/embeddingFilter.spec.ts index f893177d2a..40e957e609 100644 --- a/ts/packages/knowPro/test/embeddingFilter.spec.ts +++ b/ts/packages/knowPro/test/embeddingFilter.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { EmbeddingIndex } from "../src/fuzzyIndex.js"; describe("filtered embedding search", () => { diff --git a/ts/packages/memory/service/test/viewHistoryEntries.spec.ts b/ts/packages/memory/service/test/viewHistoryEntries.spec.ts index 5625e3f902..6a1857fa66 100644 --- a/ts/packages/memory/service/test/viewHistoryEntries.spec.ts +++ b/ts/packages/memory/service/test/viewHistoryEntries.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; From 4ad037cde29851c848206faeebfe0b8d4441a636 Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:48:51 -0700 Subject: [PATCH 04/10] Add copyright headers --- ts/packages/agent-flows/test/scriptExecutor.spec.ts | 3 +++ ts/packages/copilot-macros/test/macroGrammarCache.spec.ts | 3 +++ ts/packages/knowPro/test/embeddingFilter.spec.ts | 3 +++ ts/packages/memory/service/test/viewHistoryEntries.spec.ts | 3 +++ 4 files changed, 12 insertions(+) diff --git a/ts/packages/agent-flows/test/scriptExecutor.spec.ts b/ts/packages/agent-flows/test/scriptExecutor.spec.ts index ec6c165c41..a228146b8b 100644 --- a/ts/packages/agent-flows/test/scriptExecutor.spec.ts +++ b/ts/packages/agent-flows/test/scriptExecutor.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { jest } from "@jest/globals"; import { createScriptExecutor } from "../src/execution/scriptExecutor.js"; diff --git a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts index 8d918eee56..3e2c771da4 100644 --- a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts +++ b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import type { CopilotToolMacro } from "../src/contracts.js"; const jest = ( diff --git a/ts/packages/knowPro/test/embeddingFilter.spec.ts b/ts/packages/knowPro/test/embeddingFilter.spec.ts index f893177d2a..40e957e609 100644 --- a/ts/packages/knowPro/test/embeddingFilter.spec.ts +++ b/ts/packages/knowPro/test/embeddingFilter.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { EmbeddingIndex } from "../src/fuzzyIndex.js"; describe("filtered embedding search", () => { diff --git a/ts/packages/memory/service/test/viewHistoryEntries.spec.ts b/ts/packages/memory/service/test/viewHistoryEntries.spec.ts index 5625e3f902..6a1857fa66 100644 --- a/ts/packages/memory/service/test/viewHistoryEntries.spec.ts +++ b/ts/packages/memory/service/test/viewHistoryEntries.spec.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; From 6c2dfeafcae3c457e4c9fb2bf9d6e67e9c171e89 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Mon, 19 Oct 2026 22:59:48 +0000 Subject: [PATCH 05/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/copilot-macros/README.AUTOGEN.md | 6 ++-- ts/packages/memory/service/README.AUTOGEN.md | 32 +++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/ts/packages/copilot-macros/README.AUTOGEN.md b/ts/packages/copilot-macros/README.AUTOGEN.md index acd27f074f..d78c16f49d 100644 --- a/ts/packages/copilot-macros/README.AUTOGEN.md +++ b/ts/packages/copilot-macros/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/copilot-macros — AI-generated documentation @@ -66,10 +66,10 @@ External: `@tootallnate/quickjs-emscripten`, `typescript`, `zod` - [./src/macroExpressions.ts](./src/macroExpressions.ts) - [./src/macroFeatures.ts](./src/macroFeatures.ts) - [./src/macroLearning.ts](./src/macroLearning.ts) -- _…and 29 more under `./src/`._ +- _…and 31 more under `./src/`._ --- -_Auto-generated against commit `58e953183e0f84e370a28db042a185667f3054a1` on `2026-10-16T08:08:04.280Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/copilot-macros docs:verify-links` to spot-check._ +_Auto-generated against commit `78429ec5b77de3febc4f6d2ac9b640a83796a087` on `2026-10-19T22:57:12.751Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/copilot-macros docs:verify-links` to spot-check._ diff --git a/ts/packages/memory/service/README.AUTOGEN.md b/ts/packages/memory/service/README.AUTOGEN.md index 0ab28427cd..1b04aa52cc 100644 --- a/ts/packages/memory/service/README.AUTOGEN.md +++ b/ts/packages/memory/service/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/memory-service — AI-generated documentation @@ -22,19 +22,27 @@ Transport-independent memory corpus service - default → [./dist/index.js](./dist/index.js) - `./rpc` → [./dist/rpcFacade.js](./dist/rpcFacade.js) +- `./agent-edition-validation` → [./dist/agentEditionValidation.js](./dist/agentEditionValidation.js) +- `./procedure-markdown` → [./dist/procedureMarkdown.js](./dist/procedureMarkdown.js) +- `./procedureMarkdown` → [./dist/procedureMarkdown.js](./dist/procedureMarkdown.js) +- `./view-text` → [./dist/viewText.js](./dist/viewText.js) ### Dependencies Workspace: +- [@typeagent/aiclient](../../../packages/aiclient/README.md) +- [@typeagent/config](../../../packages/config/README.md) - [@typeagent/conversation-memory](../../../packages/memory/conversation/README.md) - [@typeagent/knowpro](../../../packages/knowPro/README.md) +- [@typeagent/textpro](../../../packages/textPro/README.md) -External: `proper-lockfile` +External: `isomorphic-git`, `node-diff3`, `proper-lockfile`, `typechat` ### Used by - [@typeagent/browser](../../../packages/agents/browser/README.md) +- [@typeagent/browser-control-rpc](../../../packages/agents/browserControlRpc/README.md) - [@typeagent/memory-agent](../../../packages/agents/memory/README.md) - [@typeagent/memory-client](../../../packages/memory/client/README.md) - [@typeagent/memory-mcp-server](../../../packages/memory/mcp-server/README.md) @@ -44,10 +52,26 @@ External: `proper-lockfile` ### Files of interest -`./src/indexSchema.ts`, `./src/index.ts`, `./src/fileMemoryService.ts`, …and 5 more under `./src/`. +- [./src/indexSchema.ts](./src/indexSchema.ts) +- [./src/index.ts](./src/index.ts) +- [./src/agentEdition.ts](./src/agentEdition.ts) +- [./src/agentEditionValidation.ts](./src/agentEditionValidation.ts) +- [./src/batchImport.ts](./src/batchImport.ts) +- [./src/boundedIngestionGroup.ts](./src/boundedIngestionGroup.ts) +- [./src/changeReceipts.ts](./src/changeReceipts.ts) +- [./src/corpusAccess.ts](./src/corpusAccess.ts) +- [./src/durableRunbookJson.ts](./src/durableRunbookJson.ts) +- [./src/eventSegmentStore.ts](./src/eventSegmentStore.ts) +- _…and 60 more under `./src/`._ + +### Environment variables + +_1 environment variable referenced from `./src/` (set in `ts/.env` or your shell). See the `## Setup` section above for guidance on obtaining each value._ + +- `TYPEAGENT_MEMORY_VIEW_MODEL_ENDPOINT` --- -_Auto-generated against commit `9fc6f4ab76b32ea5c60ad14107ee3d95c276c83b` on `2026-10-01T08:35:59.559Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/memory-service docs:verify-links` to spot-check._ +_Auto-generated against commit `78429ec5b77de3febc4f6d2ac9b640a83796a087` on `2026-10-19T22:57:12.751Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/memory-service docs:verify-links` to spot-check._ From a34a7e3a231eb9428223ca663306194f81346757 Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:00:54 -0700 Subject: [PATCH 06/10] Fixed failing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated embedding tests to verify persisted vectors through the reader instead of requiring the obsolete binary file. - Fixed segmented text-only loading in serialization.ts, which caused restart searches to report corrupt embeddings. - Fixed concurrent reindex cleanup in fileMemoryService.ts so new sources aren’t mistaken for removed sources. - Updated snapshot assertions for optimized history reads and strengthened regression coverage. --- ts/docs/architecture/memory/current-system.md | 13 ++++++++-- ts/packages/knowPro/src/serialization.ts | 2 ++ .../test/serializationIdentity.spec.ts | 2 +- .../memory/service/src/fileMemoryService.ts | 15 ++++++++++- .../test/corpusIngestionIsolation.spec.ts | 6 +++++ .../memory/service/test/snapshotReuse.spec.ts | 26 +++++++++---------- .../memory/service/test/timeline.spec.ts | 14 ++++++++-- .../test/viewSynthesisEvidence.spec.ts | 12 ++++++++- 8 files changed, 70 insertions(+), 20 deletions(-) diff --git a/ts/docs/architecture/memory/current-system.md b/ts/docs/architecture/memory/current-system.md index 4b05e77374..6a5efa0ab9 100644 --- a/ts/docs/architecture/memory/current-system.md +++ b/ts/docs/architecture/memory/current-system.md @@ -238,8 +238,11 @@ replacement does not establish power-loss durability. KnowPro derived generations publish small descriptors referencing shared immutable JSON nodes under their root's `.segments`. Copying a generation no -longer copies all retained node payloads. Compatible retain-history replacements -without acquired assets append only changed content and persist active revision +longer copies all retained node payloads. Message and related-term vectors live +in those nodes rather than a monolithic binary file. Text-only loading discards +segmented vectors along with their index metadata before identity validation. +Compatible retain-history replacements without acquired assets append only +changed content and persist active revision eligibility; inactive knowledge/vector candidates are excluded before TopN. Historical reactivation retains message ordinals without re-extraction. Replace-without-history and unsupported custom indexes still rebuild. Explicit @@ -272,6 +275,12 @@ A provider that ignores cancellation can therefore delay cleanup; the service does not report successful cleanup while that provider could still restore removed files. +Cleanup distinguishes sources captured from the committed corpus from new +sources being ingested. A concurrent reindex retains an ingestion's detached +generation until preparation finishes, then the ingestion fails its stale-index +publication check. Forget still cancels any preparation containing the forgotten +source, including an uncommitted addition, and clear cancels all preparations. + Each corpus's `views` directory contains a private bare repository, `view-history.git`. Its committed snapshots hold typed Views, generated bases, edits, conflicts, publication and maintenance state, and Topics. The ref diff --git a/ts/packages/knowPro/src/serialization.ts b/ts/packages/knowPro/src/serialization.ts index 9dc5f26c11..d11a684df9 100644 --- a/ts/packages/knowPro/src/serialization.ts +++ b/ts/packages/knowPro/src/serialization.ts @@ -119,6 +119,7 @@ export async function readConversationDataFromFile( validateFileData(fileData, embeddingSize); if (options?.loadEmbeddings === false) { dropPersistedEmbeddings(fileData.jsonData); + fileData.binaryData = {}; embeddingSize = undefined; } checkIdentities(fileData, expectedIdentities, embeddingSize); @@ -136,6 +137,7 @@ export async function readConversationDataFromFile( `knowPro: persisted embeddings (size ${getStoredEmbeddingSize(fileData)}) are incompatible with the current embedding model (size ${embeddingSize ?? "none"}). Dropping stored vectors; affected indexes will rebuild.`, ); dropPersistedEmbeddings(fileData.jsonData); + fileData.binaryData = {}; } if (embeddingsCompatible && embeddingSize && embeddingSize > 0) { if (!fileData.binaryData.embeddings) { diff --git a/ts/packages/knowPro/test/serializationIdentity.spec.ts b/ts/packages/knowPro/test/serializationIdentity.spec.ts index bb7269e131..54b6d148ba 100644 --- a/ts/packages/knowPro/test/serializationIdentity.spec.ts +++ b/ts/packages/knowPro/test/serializationIdentity.spec.ts @@ -96,7 +96,7 @@ describe("identity-aware conversation serialization", () => { directory, "known", 2, - undefined, + { message: identity }, { loadEmbeddings: false }, ); expect(text!.messageIndexData!.indexData).toBeUndefined(); diff --git a/ts/packages/memory/service/src/fileMemoryService.ts b/ts/packages/memory/service/src/fileMemoryService.ts index 90a2be6725..eb61f66451 100644 --- a/ts/packages/memory/service/src/fileMemoryService.ts +++ b/ts/packages/memory/service/src/fileMemoryService.ts @@ -1029,6 +1029,7 @@ export class FileMemoryService { corpusId: string; sourceIds: Set; + baseSourceIds: Set; controller: AbortController; prepared: Promise; jobIds?: string[]; @@ -7221,6 +7222,7 @@ export class FileMemoryService this.detachedRebuilds.set(directory, { corpusId, sourceIds, + baseSourceIds: sourceIds, controller, prepared, }); @@ -7385,7 +7387,10 @@ export class FileMemoryService directory === this.indexDirectory(corpusId, activeGeneration) || !(removal ? preparation.sourceIds.has(removal.sourceId) - : [...preparation.sourceIds].some((id) => !sources.has(id))) + : this.corpora.get(corpusId)?.manifest.pendingCorpusClear || + [...preparation.baseSourceIds].some( + (id) => !sources.has(id), + )) ) continue; preparation.controller.abort( @@ -7641,6 +7646,9 @@ export class FileMemoryService this.detachedRebuilds.set(directory, { corpusId, sourceIds, + baseSourceIds: new Set( + runtime.manifest.sources.map((source) => source.sourceId), + ), controller, prepared, jobIds: entries.map((entry) => entry.job.jobId), @@ -7932,6 +7940,11 @@ export class FileMemoryService (item) => item.sourceId, ), ), + baseSourceIds: new Set( + runtime.manifest.sources.map( + (item) => item.sourceId, + ), + ), controller: preparationController, prepared, jobIds: [job.jobId], diff --git a/ts/packages/memory/service/test/corpusIngestionIsolation.spec.ts b/ts/packages/memory/service/test/corpusIngestionIsolation.spec.ts index 923d4c2a93..61c6fef761 100644 --- a/ts/packages/memory/service/test/corpusIngestionIsolation.spec.ts +++ b/ts/packages/memory/service/test/corpusIngestionIsolation.spec.ts @@ -220,6 +220,12 @@ describe("detached corpus ingestion preparation", () => { const accepted = await append(corpusId); await provider.entered; await service.reindexCorpus(corpusId); + expect((await service.getJob(accepted.jobId))?.state).toBe( + "building-indexes", + ); + expect(await readdir(path.join(root, corpusId, "index"))).toHaveLength( + 2, + ); provider.release(); const job = await waitForMemoryJob(service, accepted.jobId); diff --git a/ts/packages/memory/service/test/snapshotReuse.spec.ts b/ts/packages/memory/service/test/snapshotReuse.spec.ts index 561c1f5410..80a14ff254 100644 --- a/ts/packages/memory/service/test/snapshotReuse.spec.ts +++ b/ts/packages/memory/service/test/snapshotReuse.spec.ts @@ -273,8 +273,8 @@ describe("operation-scoped committed view snapshots", () => { corpusId, viewId: target.viewId, }); - const oldReads = reads; - expect(oldReads).toBeGreaterThan(1); + const historicalReads = reads; + expect(historicalReads).toBe(0); expect(versions).toEqual(historical.map((entry) => entry.version)); expect( automaticallyConsolidatable( @@ -289,9 +289,9 @@ describe("operation-scoped committed view snapshots", () => { ): Promise>; }; const currentVersions = TypedViewStore.prototype.getViewVersions; - let oldPairs: Array<[ViewVersion, ViewVersion]>; - let oldNominationReads: number; - let oldNominationMs: number; + let historicalPairs: Array<[ViewVersion, ViewVersion]>; + let historicalNominationReads: number; + let historicalNominationMs: number; try { TypedViewStore.prototype.getViewVersions = async function ( request, @@ -302,16 +302,16 @@ describe("operation-scoped committed view snapshots", () => { }; reads = 0; const started = performance.now(); - oldPairs = await nomination.readConsolidationPairs( + historicalPairs = await nomination.readConsolidationPairs( corpusId, target.viewId, ); - oldNominationMs = performance.now() - started; - oldNominationReads = reads; + historicalNominationMs = performance.now() - started; + historicalNominationReads = reads; } finally { TypedViewStore.prototype.getViewVersions = currentVersions; } - expect(oldNominationReads).toBeGreaterThan(1); + expect(historicalNominationReads).toBe(1); const nominationMs: number[] = []; for (let attempt = 0; attempt < 2; attempt++) { const operation = { count: 0 }; @@ -320,7 +320,7 @@ describe("operation-scoped committed view snapshots", () => { nomination.readConsolidationPairs(corpusId, target.viewId), ); nominationMs.push(performance.now() - started); - expect(pairs).toEqual(oldPairs); + expect(pairs).toEqual(historicalPairs); expect(pairs.length).toBeGreaterThan(0); expect( pairs.some((pair) => @@ -337,9 +337,9 @@ describe("operation-scoped committed view snapshots", () => { console.warn( JSON.stringify({ snapshotReuse: { - oldHistoricalReads: oldReads, - oldNominationReads, - oldNominationMs, + historicalReads, + historicalNominationReads, + historicalNominationMs, nominationReads: 1, nominationMs, }, diff --git a/ts/packages/memory/service/test/timeline.spec.ts b/ts/packages/memory/service/test/timeline.spec.ts index 5f1007a32e..a5e155e333 100644 --- a/ts/packages/memory/service/test/timeline.spec.ts +++ b/ts/packages/memory/service/test/timeline.spec.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import type { StructuredOutputJsonSchema } from "@typeagent/aiclient"; -import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { timelineTimestamp } from "../src/timeline.js"; @@ -76,6 +76,7 @@ const { createKnowProCorpusIndex } = await import( const { createDocMemorySettings } = await import( "@typeagent/conversation-memory" ); +const { readConversationDataFromFile } = await import("@typeagent/knowpro"); const { viewHash } = await import("../src/viewMerge.js"); const { ViewHistory } = await import("../src/viewHistory.js"); @@ -1417,8 +1418,17 @@ describe("timeline configured service pipeline", () => { viewHash("indexed-timeline"), viewHash(revisionId), ); + const persisted = await readConversationDataFromFile( + directory, + "corpus", + offlineEmbedding.embeddingIdentity?.dimensions, + ); + expect( + persisted?.messageIndexData?.indexData?.embeddings.length, + ).toBeGreaterThan(0); expect( - (await stat(path.join(directory, "corpus_embeddings.bin"))).size, + persisted?.relatedTermsIndexData?.textEmbeddingData?.embeddings + .length, ).toBeGreaterThan(0); await service.close(); service = create(); diff --git a/ts/packages/memory/service/test/viewSynthesisEvidence.spec.ts b/ts/packages/memory/service/test/viewSynthesisEvidence.spec.ts index a211a52b28..a7f77566cc 100644 --- a/ts/packages/memory/service/test/viewSynthesisEvidence.spec.ts +++ b/ts/packages/memory/service/test/viewSynthesisEvidence.spec.ts @@ -63,6 +63,7 @@ const { createKnowProCorpusIndex } = await import( const { createDocMemorySettings } = await import( "@typeagent/conversation-memory" ); +const { readConversationDataFromFile } = await import("@typeagent/knowpro"); const { viewHash } = await import("../src/viewMerge.js"); const { effectiveViewPublicationPolicy } = await import( "../src/viewPublication.js" @@ -637,8 +638,17 @@ describe("actual configured evidence-first adapter and durable service", () => { viewHash("guide"), viewHash(revisionId), ); + const persisted = await readConversationDataFromFile( + directory, + "corpus", + offlineEmbedding.embeddingIdentity?.dimensions, + ); + expect( + persisted?.messageIndexData?.indexData?.embeddings.length, + ).toBeGreaterThan(0); expect( - (await stat(path.join(directory, "corpus_embeddings.bin"))).size, + persisted?.relatedTermsIndexData?.textEmbeddingData?.embeddings + .length, ).toBeGreaterThan(0); await service.close(); service = create(); From d04461f9884a49bc62865de5b55a8759b0db6614 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Tue, 20 Oct 2026 16:29:13 +0000 Subject: [PATCH 07/10] style: apply prettier formatting and policy fixes --- ts/packages/copilot-macros/test/macroPrograms.spec.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ts/packages/copilot-macros/test/macroPrograms.spec.ts b/ts/packages/copilot-macros/test/macroPrograms.spec.ts index a6746cc46a..f97b215261 100644 --- a/ts/packages/copilot-macros/test/macroPrograms.spec.ts +++ b/ts/packages/copilot-macros/test/macroPrograms.spec.ts @@ -9,7 +9,15 @@ import { type RecordedInteractionTrace, } from "@typeagent/copilot-macros"; import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, readdir, rename, rm, rmdir } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + rmdir, +} from "node:fs/promises"; import path from "node:path"; import { compileRecipeProgram } from "../src/recipeCompiler.js"; import { RecipeExecutor } from "../src/recipeExecutor.js"; From 6c5d8dc89190a3602376bdb4cc9d4b87930587ac Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:32:47 -0700 Subject: [PATCH 08/10] Fix build failures --- .../test/macroChildAuthoring.spec.ts | 1 - .../test/macroGrammarCache.spec.ts | 6 ++++ .../copilot-macros/test/macroPrograms.spec.ts | 28 ++++++++----------- .../src/extension/macro-authoring.ts | 2 +- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts b/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts index 0efa0dad39..a86c58bce1 100644 --- a/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts +++ b/ts/packages/copilot-macros/test/macroChildAuthoring.spec.ts @@ -2,7 +2,6 @@ // Licensed under the MIT License. import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; import path from "node:path"; import type { MacroLearningJob, diff --git a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts index 3e2c771da4..f4370b6cdf 100644 --- a/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts +++ b/ts/packages/copilot-macros/test/macroGrammarCache.spec.ts @@ -35,6 +35,12 @@ function macro(phrase: string): CopilotToolMacro { createdAt: "2026-10-18T00:00:00Z", warnings: [], learning: { + authoring: { + captureId: "fixture", + taskId: "fixture:root-user", + tokenId: "fixture-token", + contextDigest: "a".repeat(64), + }, jobId: "fixture", cwd: ".", mode: "read-only", diff --git a/ts/packages/copilot-macros/test/macroPrograms.spec.ts b/ts/packages/copilot-macros/test/macroPrograms.spec.ts index f97b215261..c54474d2f3 100644 --- a/ts/packages/copilot-macros/test/macroPrograms.spec.ts +++ b/ts/packages/copilot-macros/test/macroPrograms.spec.ts @@ -9,15 +9,7 @@ import { type RecordedInteractionTrace, } from "@typeagent/copilot-macros"; import { createHash } from "node:crypto"; -import { - mkdir, - mkdtemp, - readFile, - readdir, - rename, - rm, - rmdir, -} from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, rename, rm, rmdir } from "node:fs/promises"; import path from "node:path"; import { compileRecipeProgram } from "../src/recipeCompiler.js"; import { RecipeExecutor } from "../src/recipeExecutor.js"; @@ -603,13 +595,17 @@ describe("TypeAgent program macro integration", () => { it("rejects a missing runtime without falling back", async () => { const { manager, ref } = await setup(); - await expect( - manager.runMacro({ - ...ref, - runId: "missing", - inputs: { step_1_path: "beta.txt" }, - }), - ).rejects.toThrow("runtime is unavailable"); + try { + await expect( + manager.runMacro({ + ...ref, + runId: "missing", + inputs: { step_1_path: "beta.txt" }, + }), + ).rejects.toThrow("runtime is unavailable"); + } finally { + await manager.close(); + } }); it("pins authoritative revisions and rejects duplicate run IDs", async () => { diff --git a/ts/packages/copilot-plugin/src/extension/macro-authoring.ts b/ts/packages/copilot-plugin/src/extension/macro-authoring.ts index 56549266d9..4293ebf4d3 100644 --- a/ts/packages/copilot-plugin/src/extension/macro-authoring.ts +++ b/ts/packages/copilot-plugin/src/extension/macro-authoring.ts @@ -9,7 +9,7 @@ import type { import { randomUUID } from "node:crypto"; import { redactTraceValue } from "@typeagent/copilot-macros"; -function waitForAuthoringPoll(signal: AbortSignal): Promise { +function waitForAuthoringReconnect(signal: AbortSignal): Promise { return new Promise((resolve) => { const done = () => { clearTimeout(timer); From 72c81823b768200e3216d849dde84eef41e18111 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Tue, 20 Oct 2026 17:43:08 +0000 Subject: [PATCH 09/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/copilot-macros/README.AUTOGEN.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ts/packages/copilot-macros/README.AUTOGEN.md b/ts/packages/copilot-macros/README.AUTOGEN.md index d78c16f49d..dd2aaa246e 100644 --- a/ts/packages/copilot-macros/README.AUTOGEN.md +++ b/ts/packages/copilot-macros/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/copilot-macros — AI-generated documentation @@ -48,10 +48,8 @@ External: `@tootallnate/quickjs-emscripten`, `typescript`, `zod` ### Used by -- [@typeagent/agent-server-protocol](../../packages/agentServer/protocol/README.md) - [@typeagent/copilot-plugin](../../packages/copilot-plugin/README.md) - [@typeagent/procedure-artifacts](../../packages/procedureArtifacts/README.md) -- [agent-server](../../packages/agentServer/server/README.md) - [default-agent-provider](../../packages/defaultAgentProvider/README.md) ### Files of interest @@ -70,6 +68,6 @@ External: `@tootallnate/quickjs-emscripten`, `typescript`, `zod` --- -_Auto-generated against commit `78429ec5b77de3febc4f6d2ac9b640a83796a087` on `2026-10-19T22:57:12.751Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/copilot-macros docs:verify-links` to spot-check._ +_Auto-generated against commit `6c5d8dc89190a3602376bdb4cc9d4b87930587ac` on `2026-10-20T17:40:06.178Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/copilot-macros docs:verify-links` to spot-check._ From 04e9bba38f4af232a627c7e4156c640a43dfecba Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:05:27 -0700 Subject: [PATCH 10/10] fix: persist rejected learning jobs and stabilize CI tests --- ts/packages/copilot-macros/README.md | 6 ++ .../copilot-macros/src/macroLearning.ts | 19 +++--- .../copilot-macros/test/macroLearning.spec.ts | 58 +++++++++---------- .../test/controlMetadata.spec.ts | 41 ++++++++----- 4 files changed, 71 insertions(+), 53 deletions(-) diff --git a/ts/packages/copilot-macros/README.md b/ts/packages/copilot-macros/README.md index 80c1a3d7b4..d5228fdef2 100644 --- a/ts/packages/copilot-macros/README.md +++ b/ts/packages/copilot-macros/README.md @@ -484,6 +484,12 @@ The focused specs cover definition and validation, contained program execution, catalog lifecycle, approval, persistence, cancellation, schema drift, and run-record sanitization. +Persistence fixtures use `RevisionRecords` to load and commit learning jobs by +job ID. `learning.json` is a revision manifest, not an inline `jobs` array; +editing the manifest directly does not update its immutable job records. +Restart validation persists rejected child contexts as failed job revisions, +including the instruction to re-record the task. + ## Limitations - Single-trace induction is deliberately narrow. diff --git a/ts/packages/copilot-macros/src/macroLearning.ts b/ts/packages/copilot-macros/src/macroLearning.ts index 7b8751d73c..911cbcbbb8 100644 --- a/ts/packages/copilot-macros/src/macroLearning.ts +++ b/ts/packages/copilot-macros/src/macroLearning.ts @@ -1189,21 +1189,26 @@ export class MacroLearningEngine { ) { throw new Error("Invalid durable learning state."); } - let rejected = false; + const rejected: StoredJob[] = []; for (const job of this.state.jobs) { try { if (!validAuthoringIdentity(job)) throw new Error("Missing child identity."); await this.childAuthoring.assertContext(job); } catch { - job.status = "failed"; - job.error = - "Unsupported learning job: sealed child context is missing or changed; re-record the successful task."; - job.updatedAt = new Date().toISOString(); - rejected = true; + rejected.push(job); } } - if (rejected) await this.update(() => {}); + if (rejected.length) + await this.update(() => { + for (const job of rejected) { + this.markChanged(job); + job.status = "failed"; + job.error = + "Unsupported learning job: sealed child context is missing or changed; re-record the successful task."; + job.updatedAt = new Date().toISOString(); + } + }); function validAuthoringIdentity(job: StoredJob): boolean { const identity = job.authoring; diff --git a/ts/packages/copilot-macros/test/macroLearning.spec.ts b/ts/packages/copilot-macros/test/macroLearning.spec.ts index 4ad8409ec8..c550924471 100644 --- a/ts/packages/copilot-macros/test/macroLearning.spec.ts +++ b/ts/packages/copilot-macros/test/macroLearning.spec.ts @@ -61,19 +61,23 @@ async function setup(mode: MacroLearningMode = "prepare", readOnly?: boolean) { return { directory, manager, host }; } +type StoredTestJob = Omit & { + authoring?: MacroLearningJob["authoring"]; + attempts: Record; + draft?: unknown; +}; + async function mutateStoredJob( directory: string, jobId: string, - mutate: (job: { status: string; attempts: Record }) => void, + mutate: (job: StoredTestJob) => void, ) { const records = new RevisionRecords( path.join(directory, "copilot-macros", "learning.json"), ); const state = await records.load(); - const job = state.get(jobId) as { - status: string; - attempts: Record; - }; + const job = state.get(jobId) as StoredTestJob; + expect(job).toBeDefined(); mutate(job); await records.commit(new Map([[jobId, job]])); } @@ -244,15 +248,14 @@ describe("durable macro learning", () => { "copilot-macros", "learning.json", ); - const stored = JSON.parse(await readFile(filename, "utf8")); - const job = stored.jobs[0]; - delete job.authoring; - job.status = status === "saved-catalog" ? "failed" : status; - if (status === "saved-catalog") { - job.draft = { old: true }; - job.error = "catalog was saved"; - } - await writeFile(filename, JSON.stringify(stored)); + await mutateStoredJob(directory, summary.learningJobId!, (job) => { + delete job.authoring; + job.status = status === "saved-catalog" ? "failed" : status; + if (status === "saved-catalog") { + job.draft = { old: true }; + job.error = "catalog was saved"; + } + }); const restarted = createManager(directory, host); const learner = runtime(); await restarted.configureLearning(learner); @@ -265,9 +268,8 @@ describe("durable macro learning", () => { expect(learner.extract).not.toHaveBeenCalled(); expect(learner.build).not.toHaveBeenCalled(); await restarted.close(); - expect( - JSON.parse(await readFile(filename, "utf8")).jobs[0], - ).toMatchObject({ + const persisted = await new RevisionRecords(filename).load(); + expect(persisted.get(summary.learningJobId!)).toMatchObject({ status: "failed", error: expect.stringContaining("re-record"), }); @@ -293,14 +295,13 @@ describe("durable macro learning", () => { }, ); } else { - const filename = path.join( + await mutateStoredJob( directory, - "copilot-macros", - "learning.json", + summary.learningJobId!, + (job) => { + job.authoring!.contextDigest = "0".repeat(64); + }, ); - const stored = JSON.parse(await readFile(filename, "utf8")); - stored.jobs[0].authoring.contextDigest = "0".repeat(64); - await writeFile(filename, JSON.stringify(stored)); } const restarted = createManager(directory, host); const learner = runtime(); @@ -370,14 +371,9 @@ describe("durable macro learning", () => { const staged = await terminal(manager, summary.learningJobId!); expect(staged.status).toBe("needsReview"); await manager.close(); - const filename = path.join( - directory, - "copilot-macros", - "learning.json", - ); - const stored = JSON.parse(await readFile(filename, "utf8")); - delete stored.jobs[0].authoring; - await writeFile(filename, JSON.stringify(stored)); + await mutateStoredJob(directory, summary.learningJobId!, (job) => { + delete job.authoring; + }); const restarted = createManager(directory, host); await restarted.configureLearning(runtime()); await expect(restarted.approveMacro(staged.macro!)).rejects.toThrow( diff --git a/ts/packages/copilot-plugin/test/controlMetadata.spec.ts b/ts/packages/copilot-plugin/test/controlMetadata.spec.ts index 04da150524..66dd9c3d5e 100644 --- a/ts/packages/copilot-plugin/test/controlMetadata.spec.ts +++ b/ts/packages/copilot-plugin/test/controlMetadata.spec.ts @@ -8,8 +8,10 @@ import { rmSync, readFileSync, readdirSync, + watchFile, unwatchFile, } from "node:fs"; +import { once } from "node:events"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { @@ -439,15 +441,18 @@ describe("owned control metadata", () => { diagnosticTransitions: true, }), ); - writeFileSync( - join(root, "plugin-data", "config.json"), - JSON.stringify({ mode: "bypass" }), + const configPath = join(root, "plugin-data", "config.json"); + // A missing-file notification confirms the poller's initial stat completed. + const watcher = watchFile( + configPath, + { interval: 200, persistent: false }, + () => {}, ); + const watching = once(watcher, "change", { + signal: AbortSignal.timeout(5000), + }); monitorControlMetadata("mcp-agent"); - writeFileSync( - join(root, "plugin-data", "config.json"), - JSON.stringify({ mode: "mcp", mcpRouting: "mixed" }), - ); + await watching; const receiptFile = join( root, "data", @@ -455,15 +460,21 @@ describe("owned control metadata", () => { `mcp-agent-${process.pid}.json`, ); const readReceipt = () => JSON.parse(readFileSync(receiptFile, "utf8")); - const deadline = Date.now() + 5000; - let receipt = readReceipt(); - while (receipt.effectiveMode !== "mcp" && Date.now() < deadline) { - await delay(20); - receipt = readReceipt(); + for (const mode of ["bypass", "mcp"]) { + writeFileSync( + configPath, + JSON.stringify({ mode, mcpRouting: "mixed" }), + ); + const deadline = Date.now() + 5000; + let receipt = readReceipt(); + while (receipt.effectiveMode !== mode && Date.now() < deadline) { + await delay(20); + receipt = readReceipt(); + } + expect(receipt.effectiveMode).toBe(mode); + expect(receipt.source.routing).toBe("mixed"); + expect(receipt.source.request).toBeNull(); } - expect(receipt.effectiveMode).toBe("mcp"); - expect(receipt.source.routing).toBe("mixed"); - expect(receipt.source.request).toBeNull(); }); it.each([ "exact-docs",