diff --git a/apps/server/src/modules/agent/acp/external-agent-realization.ts b/apps/server/src/modules/agent/acp/external-agent-realization.ts index c99542d38..0a9d4b25d 100644 --- a/apps/server/src/modules/agent/acp/external-agent-realization.ts +++ b/apps/server/src/modules/agent/acp/external-agent-realization.ts @@ -78,8 +78,10 @@ interface RealizationDependencies { readRecord: ( namespace: Namespace, threadId: string, - ) => ReturnType; - createHandle: (spec: AcpWorkloadSpec) => AcpHandle; + ) => + | ReturnType + | Awaited>; + createHandle: (spec: AcpWorkloadSpec) => AcpHandle | Promise; buildSpec: typeof buildAcpWorkloadSpec; subscribeProfileCache: typeof ensureProfileCacheSubscription; ensureSession: ( @@ -107,7 +109,7 @@ async function ensureSessionFromCanonicalSpec( resolvedEnvironment || spec.spec.env ? { ...resolvedEnvironment, ...spec.spec.env } : undefined; - const record = agenetes.record(spec.namespace, spec.threadId); + const record = await agenetes.record(spec.namespace, spec.threadId); return ensureAcpSession({ agentletId: resolveAcpAgentletId(spec), threadId: spec.threadId, @@ -135,8 +137,9 @@ const DEFAULT_DEPENDENCIES: RealizationDependencies = { resolveFixedAgentNode: (canvasId, threadId) => agentThreadResolver.resolveFixedAgentNode(canvasId, threadId), collectSpacePrompt: resolveSpacePrompt, - readRecord: (namespace, threadId) => agenetes.record(namespace, threadId), - createHandle: (spec) => agenetes.create(spec) as AcpHandle, + readRecord: async (namespace, threadId) => + await agenetes.record(namespace, threadId), + createHandle: async (spec) => (await agenetes.create(spec)) as AcpHandle, buildSpec: buildAcpWorkloadSpec, subscribeProfileCache: ensureProfileCacheSubscription, ensureSession: ensureSessionFromCanonicalSpec, @@ -201,7 +204,10 @@ export class ExternalAgentRealizationService { ) : null)) : options.agentTarget; - const record = this.dependencies.readRecord(namespace, options.threadId); + const record = await this.dependencies.readRecord( + namespace, + options.threadId, + ); if (record) { if (record.spec.kind !== EXTERNAL_DRIVER_KIND) { @@ -216,7 +222,7 @@ export class ExternalAgentRealizationService { binding, fixedTarget, spec, - handle: this.dependencies.createHandle(spec), + handle: await this.dependencies.createHandle(spec), }; this.dependencies.subscribeProfileCache( options.threadId, @@ -293,7 +299,7 @@ export class ExternalAgentRealizationService { binding, fixedTarget, spec, - handle: this.dependencies.createHandle(spec), + handle: await this.dependencies.createHandle(spec), }; this.dependencies.subscribeProfileCache( options.threadId, diff --git a/apps/server/src/modules/agent/acp/threads.route.ts b/apps/server/src/modules/agent/acp/threads.route.ts index a577b1d1b..8d2977459 100644 --- a/apps/server/src/modules/agent/acp/threads.route.ts +++ b/apps/server/src/modules/agent/acp/threads.route.ts @@ -82,9 +82,15 @@ async function realizeControlThread( } } -function resolveThreadAgentletId(threadId: string, canvasId?: string): string { +async function resolveThreadAgentletId( + threadId: string, + canvasId?: string, +): Promise { if (canvasId) { - const record = agenetes.record(canvasAcpNamespace(canvasId), threadId); + const record = await agenetes.record( + canvasAcpNamespace(canvasId), + threadId, + ); const driverSpec = record?.spec.spec; if ( driverSpec && @@ -223,7 +229,7 @@ const acpThreadsRoutes: FastifyPluginAsync = async (app) => { }); } const { canvasId, profileId } = parsed.data; - const agentletId = resolveThreadAgentletId(threadId, canvasId); + const agentletId = await resolveThreadAgentletId(threadId, canvasId); const live = acpSessionRegistry.get(agentletId, threadId); if (live) { return { @@ -234,7 +240,10 @@ const acpThreadsRoutes: FastifyPluginAsync = async (app) => { }; } if (canvasId) { - const record = agenetes.record(canvasAcpNamespace(canvasId), threadId); + const record = await agenetes.record( + canvasAcpNamespace(canvasId), + threadId, + ); const persistedMeta = record?.state?.metadata; if (persistedMeta) { return { diff --git a/apps/server/src/modules/agent/agenetes/conversation-stores.ts b/apps/server/src/modules/agent/agenetes/conversation-stores.ts index 2c235ed4d..ed18171ed 100644 --- a/apps/server/src/modules/agent/agenetes/conversation-stores.ts +++ b/apps/server/src/modules/agent/agenetes/conversation-stores.ts @@ -30,20 +30,23 @@ import { } from '@agenetes/agenetes'; import { - conversationTables, + PostgresThreadStore, + PostgresEventLogStore, + PostgresTurnStore, +} from './postgres-stores.js'; +import { SqliteEventLogStore, SqliteThreadStore, SqliteTurnStore, } from './sqlite-stores.js'; +import { getStructuredStore } from '../../storage/index.js'; import type { - EventLogEntry, EventLogRecord, EventLogStore, PersistedTurn, ThreadRecord, ThreadStore, - TurnStartLogEntry, TurnStore, } from '@agenetes/agenetes'; import type { AgentSubmission, Namespace } from '@agenetes/protocol'; @@ -60,6 +63,12 @@ const file: Backing = { turns: new FileTurnStore(), }; +const postgres: Backing = { + threads: new PostgresThreadStore(), + events: new PostgresEventLogStore(), + turns: new PostgresTurnStore(), +}; + const sqlite: Backing = { threads: new SqliteThreadStore(), events: new SqliteEventLogStore(), @@ -81,8 +90,11 @@ function backingFor(namespace: Namespace): Backing { // A directory to write into settles it: that is the Disk profile, and the // file stores are what wrote whatever is already there. if (namespace.storage?.root) return file; - if (namespace.name && conversationTables(namespace) !== null) return sqlite; - return memory; + if (!namespace.name) return memory; + const kind = getStructuredStore().kind; + if (kind === 'postgres') return postgres; + if (kind === 'sqlite') return sqlite; + throw new Error('A named Disk conversation requires a storage root'); } export const conversationThreadStore: ThreadStore = { @@ -96,13 +108,9 @@ export const conversationThreadStore: ThreadStore = { }; export const conversationEventLogStore: EventLogStore = { - appendTurnStart: ( - namespace, - threadId, - request: AgentSubmission | null, - ): TurnStartLogEntry => + appendTurnStart: (namespace, threadId, request: AgentSubmission | null) => backingFor(namespace).events.appendTurnStart(namespace, threadId, request), - append: (namespace, threadId, event): EventLogEntry => + append: (namespace, threadId, event) => backingFor(namespace).events.append(namespace, threadId, event), read: (namespace, threadId, sinceSeq) => backingFor(namespace).events.read(namespace, threadId, sinceSeq), diff --git a/apps/server/src/modules/agent/agenetes/postgres-stores.ts b/apps/server/src/modules/agent/agenetes/postgres-stores.ts new file mode 100644 index 000000000..cbfbd3dfa --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/postgres-stores.ts @@ -0,0 +1,362 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Conversation tables remain owned by Agenetes, attached to storage's parent row. */ +import { space } from '../../storage/index.js'; + +import type { + EventLogEntry, + EventLogRecord, + EventLogStore, + PersistedTurn, + ThreadRecord, + ThreadStore, + TurnStartLogEntry, + TurnStore, +} from '@agenetes/agenetes'; +import type { AgentSubmission, Namespace } from '@agenetes/protocol'; +import type { Pool, PoolClient } from 'pg'; + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS agenetes_threads ( + extension_id INTEGER NOT NULL REFERENCES space_extensions(extension_id) ON DELETE CASCADE, + thread_id TEXT NOT NULL, record_json TEXT NOT NULL CHECK ((record_json::json) IS NOT NULL), PRIMARY KEY(extension_id, thread_id)); +CREATE TABLE IF NOT EXISTS agenetes_events ( + extension_id INTEGER NOT NULL REFERENCES space_extensions(extension_id) ON DELETE CASCADE, + thread_id TEXT NOT NULL, seq INTEGER NOT NULL, record_json TEXT NOT NULL CHECK ((record_json::json) IS NOT NULL), PRIMARY KEY(extension_id, thread_id, seq)); +CREATE TABLE IF NOT EXISTS agenetes_turns ( + extension_id INTEGER NOT NULL REFERENCES space_extensions(extension_id) ON DELETE CASCADE, + thread_id TEXT NOT NULL, ordinal INTEGER NOT NULL, seq_start INTEGER NOT NULL, seq_end INTEGER NOT NULL, + turn_json TEXT NOT NULL CHECK ((turn_json::json) IS NOT NULL), PRIMARY KEY(extension_id, thread_id, ordinal));`; +const prepared = new WeakMap>(); +async function tables(database: Pool): Promise { + let pending = prepared.get(database); + if (!pending) { + pending = (async () => { + const client = await database.connect(); + let broken = false; + try { + await client.query('BEGIN'); + await client.query("SET LOCAL lock_timeout = '5s'"); + await client.query('SELECT pg_advisory_xact_lock(184202607)'); + await client.query(SCHEMA); + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + broken = true; + } + throw error; + } finally { + client.release(broken); + } + })(); + prepared.set(database, pending); + void pending.catch(() => prepared.delete(database)); + } + await pending; +} +async function substrate(namespace: Namespace) { + if (!namespace.name) return null; + const value = await space(namespace.name).extension('agenetes.conversations'); + if (!value) return null; + if (value.kind !== 'postgres') + throw new Error('Expected Postgres conversation substrate'); + await tables(value.database); + return value; +} +async function read( + namespace: Namespace, + missing: T, + operation: (database: Pool, id: number) => Promise, +): Promise { + const value = await substrate(namespace); + return value ? operation(value.database, value.extensionId) : missing; +} +async function mutate( + namespace: Namespace, + threadId: string, + operation: (database: PoolClient, id: number) => Promise, +): Promise { + const value = await substrate(namespace); + if (!value) throw new Error('Cannot persist conversation in a missing Space'); + const client = await value.database.connect(); + let broken = false; + try { + await client.query('BEGIN'); + await client.query("SET LOCAL lock_timeout = '5s'"); + await client.query( + 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', + [`agenetes:${value.extensionId}:${threadId}`], + ); + const result = await operation(client, value.extensionId); + await client.query('COMMIT'); + return result; + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + broken = true; + } + throw error; + } finally { + client.release(broken); + } +} + +export class PostgresThreadStore implements ThreadStore { + async upsert( + namespace: Namespace, + threadId: string, + record: ThreadRecord, + ): Promise { + await mutate(namespace, threadId, async (db, id) => { + await db.query( + `INSERT INTO agenetes_threads VALUES ($1, $2, $3) + ON CONFLICT(extension_id, thread_id) DO UPDATE SET record_json = excluded.record_json`, + [id, threadId, JSON.stringify(record)], + ); + }); + } + async get( + namespace: Namespace, + threadId: string, + ): Promise { + return read(namespace, undefined, async (db, id) => { + const row = ( + await db.query( + 'SELECT record_json FROM agenetes_threads WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ) + ).rows[0]; + return row ? (JSON.parse(row.record_json) as ThreadRecord) : undefined; + }); + } + async list(namespace: Namespace): Promise { + return read(namespace, [], async (db, id) => + ( + await db.query( + 'SELECT record_json FROM agenetes_threads WHERE extension_id=$1 ORDER BY thread_id', + [id], + ) + ).rows.map((row) => JSON.parse(row.record_json) as ThreadRecord), + ); + } + async delete(namespace: Namespace, threadId: string): Promise { + if (!(await substrate(namespace))) return; + await mutate(namespace, threadId, async (db, id) => { + await db.query( + 'DELETE FROM agenetes_threads WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ); + }); + } +} + +export class PostgresEventLogStore implements EventLogStore { + private async appendRecord( + namespace: Namespace, + threadId: string, + record: Omit | Omit, + ): Promise { + return mutate(namespace, threadId, async (db, id) => { + const max = ( + await db.query( + 'SELECT COALESCE(MAX(seq),0) AS seq FROM agenetes_events WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ) + ).rows[0].seq; + const entry = { ...record, seq: Number(max) + 1 }; + await db.query('INSERT INTO agenetes_events VALUES ($1,$2,$3,$4)', [ + id, + threadId, + entry.seq, + JSON.stringify(entry), + ]); + return entry; + }); + } + async appendTurnStart( + namespace: Namespace, + threadId: string, + request: AgentSubmission | null, + ): Promise { + return (await this.appendRecord(namespace, threadId, { + kind: 'turn_start', + request, + ts: Date.now(), + })) as TurnStartLogEntry; + } + async append( + namespace: Namespace, + threadId: string, + event: EventLogEntry['event'], + ): Promise { + return (await this.appendRecord(namespace, threadId, { + event, + ts: Date.now(), + })) as EventLogEntry; + } + async readRecords( + namespace: Namespace, + threadId: string, + sinceSeq = 0, + ): Promise { + return read(namespace, [], async (db, id) => + ( + await db.query( + 'SELECT record_json FROM agenetes_events WHERE extension_id=$1 AND thread_id=$2 AND seq>$3 ORDER BY seq', + [id, threadId, sinceSeq], + ) + ).rows.map((row) => JSON.parse(row.record_json) as EventLogRecord), + ); + } + async read( + namespace: Namespace, + threadId: string, + sinceSeq = 0, + ): Promise { + return (await this.readRecords(namespace, threadId, sinceSeq)).filter( + (row): row is EventLogEntry => !('kind' in row), + ); + } + async maxSeq(namespace: Namespace, threadId: string): Promise { + return read(namespace, 0, async (db, id) => + Number( + ( + await db.query( + 'SELECT COALESCE(MAX(seq),0) AS seq FROM agenetes_events WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ) + ).rows[0].seq, + ), + ); + } + async replace( + namespace: Namespace, + threadId: string, + records: readonly EventLogRecord[], + ): Promise { + await mutate(namespace, threadId, async (db, id) => { + await db.query( + 'DELETE FROM agenetes_events WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ); + for (const row of records) + await db.query('INSERT INTO agenetes_events VALUES ($1,$2,$3,$4)', [ + id, + threadId, + row.seq, + JSON.stringify(row), + ]); + }); + } + async delete(namespace: Namespace, threadId: string): Promise { + if (!(await substrate(namespace))) return; + await mutate(namespace, threadId, async (db, id) => { + await db.query( + 'DELETE FROM agenetes_events WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ); + }); + } +} + +export class PostgresTurnStore implements TurnStore { + async append( + namespace: Namespace, + threadId: string, + record: PersistedTurn, + ): Promise { + await mutate(namespace, threadId, async (db, id) => { + const max = Number( + ( + await db.query( + 'SELECT COALESCE(MAX(ordinal),0) AS ordinal FROM agenetes_turns WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ) + ).rows[0].ordinal, + ); + await db.query('INSERT INTO agenetes_turns VALUES ($1,$2,$3,$4,$5,$6)', [ + id, + threadId, + max + 1, + record.seqStart, + record.seqEnd, + JSON.stringify(record.turn), + ]); + }); + } + async list(namespace: Namespace, threadId: string): Promise { + return read(namespace, [], async (db, id) => + ( + await db.query( + 'SELECT seq_start, seq_end, turn_json FROM agenetes_turns WHERE extension_id=$1 AND thread_id=$2 ORDER BY ordinal', + [id, threadId], + ) + ).rows.map((row) => ({ + seqStart: Number(row.seq_start), + seqEnd: Number(row.seq_end), + turn: JSON.parse(row.turn_json) as PersistedTurn['turn'], + })), + ); + } + async count(namespace: Namespace, threadId: string): Promise { + return read(namespace, 0, async (db, id) => + Number( + ( + await db.query( + 'SELECT COUNT(*) AS count FROM agenetes_turns WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ) + ).rows[0].count, + ), + ); + } + async fence(namespace: Namespace, threadId: string): Promise { + return read(namespace, 0, async (db, id) => + Number( + ( + await db.query( + 'SELECT seq_end FROM agenetes_turns WHERE extension_id=$1 AND thread_id=$2 ORDER BY ordinal DESC LIMIT 1', + [id, threadId], + ) + ).rows[0]?.seq_end ?? 0, + ), + ); + } + async replace( + namespace: Namespace, + threadId: string, + records: readonly PersistedTurn[], + ): Promise { + await mutate(namespace, threadId, async (db, id) => { + await db.query( + 'DELETE FROM agenetes_turns WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ); + for (const [index, row] of records.entries()) + await db.query( + 'INSERT INTO agenetes_turns VALUES ($1,$2,$3,$4,$5,$6)', + [ + id, + threadId, + index + 1, + row.seqStart, + row.seqEnd, + JSON.stringify(row.turn), + ], + ); + }); + } + async delete(namespace: Namespace, threadId: string): Promise { + if (!(await substrate(namespace))) return; + await mutate(namespace, threadId, async (db, id) => { + await db.query( + 'DELETE FROM agenetes_turns WHERE extension_id=$1 AND thread_id=$2', + [id, threadId], + ); + }); + } +} diff --git a/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts index dd194f8e7..8123ca82c 100644 --- a/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts +++ b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts @@ -2,7 +2,7 @@ // Licensed under the MIT license. /** - * The Agenetes conversation stores against a real SQLite profile. + * The Agenetes conversation stores against a real SQL profiles. * * The claim under test is the one a user would notice: a conversation held in * a Space that has no directory survives a restart, and goes away with its @@ -12,7 +12,7 @@ import { mountAgenetes } from '@agenetes/agenetes'; import { defineDriver } from '@agenetes/runtime'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { z } from 'zod'; import { @@ -21,12 +21,18 @@ import { conversationTurnStore, } from './conversation-stores.js'; import { conversationTables } from './sqlite-stores.js'; -import { deleteSpace } from '../../storage/index.js'; +import { deleteSpace, space } from '../../storage/index.js'; import { mountTestWorkspace, + PRODUCT_STORAGE_PROFILES, type MountedTestStorage, } from '../../storage/testing.js'; import { canvasAcpNamespace } from '../../workspace/paths.js'; +import { + readSubstrateDocument, + writeSubstrateDocument, + appendSubstrateLog, +} from '../substrate-store.js'; import type { StorageProfile } from '../../storage/profile.js'; import type { @@ -35,11 +41,9 @@ import type { ThreadRecord, } from '@agenetes/agenetes'; import type { AgentStateSnapshot, WorkloadSpec } from '@agenetes/protocol'; +import type { AgentHandle } from '@agenetes/runtime'; -const SQLITE: StorageProfile = { - structured: { kind: 'sqlite' }, - blobs: { kind: 'disk' }, -}; +let profile: StorageProfile; const CANVAS_ID = 'canvas-conversation'; const THREAD_ID = 'thread-1'; @@ -53,7 +57,7 @@ afterEach(async () => { /** Open the profile and create the Space the conversation belongs to. */ async function openWithSpace(): Promise { - const opened = await mountTestWorkspace(SQLITE, 'huabu-agenetes-sqlite-'); + const opened = await mountTestWorkspace(profile, 'huabu-agenetes-sql-'); mounted = opened; const created = await opened.storage.structured .spaces() @@ -74,17 +78,67 @@ function threadRecord(threadId = THREAD_ID): ThreadRecord { }; } -describe('Agenetes conversation stores on SQLite', () => { +async function conversationSubstrate( + namespace: ReturnType, +) { + // A store read creates its owner tables before fault injection. + await conversationThreadStore.list(namespace); + const value = + profile.structured.kind === 'sqlite' + ? conversationTables(namespace) + : await space(namespace.name).extension('agenetes.conversations'); + if (value?.kind === 'disk') + throw new Error('Expected SQL conversation tables'); + return value; +} + +async function rejectWrite( + substrate: NonNullable>>, + name: string, + table: string, + operation: string, + condition: string, + message: string, +): Promise<() => Promise> { + if (substrate.kind === 'sqlite') { + substrate.database + .exec(`CREATE TRIGGER ${name} BEFORE ${operation} ON agenetes_${table} + WHEN ${condition} BEGIN SELECT RAISE(ABORT, '${message}'); END;`); + return async () => { + substrate.database.exec(`DROP TRIGGER ${name}`); + }; + } + if (substrate.kind !== 'postgres') + throw new Error('Expected a SQL substrate'); + await substrate.database + .query(`CREATE FUNCTION ${name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF ${condition} THEN RAISE EXCEPTION '${message}'; END IF; + RETURN ${operation === 'DELETE' ? 'OLD' : 'NEW'}; END $$; + CREATE TRIGGER ${name} BEFORE ${operation} ON agenetes_${table} + FOR EACH ROW EXECUTE FUNCTION ${name}();`); + return async () => { + await substrate.database.query( + `DROP TRIGGER ${name} ON agenetes_${table}; DROP FUNCTION ${name}()`, + ); + }; +} + +describe.each( + PRODUCT_STORAGE_PROFILES.filter((p) => p.structured.kind !== 'disk'), +)('Agenetes conversation stores on %j', (selected) => { + beforeEach(() => { + profile = selected; + }); for (const kind of ['events', 'turns'] as const) { describe(`${kind} replacement`, () => { async function setupReplacement() { const opened = await openWithSpace(); const namespace = canvasAcpNamespace(CANVAS_ID); - const substrate = conversationTables(namespace); + const substrate = await conversationSubstrate(namespace); if (!substrate) throw new Error('Expected SQLite conversation tables'); - const replace = (values: readonly number[]) => { + const replace = async (values: readonly number[]) => { if (kind === 'events') { - conversationEventLogStore.replace( + await conversationEventLogStore.replace( namespace, THREAD_ID, values.map((seq) => ({ @@ -95,7 +149,7 @@ describe('Agenetes conversation stores on SQLite', () => { })), ); } else { - conversationTurnStore.replace( + await conversationTurnStore.replace( namespace, THREAD_ID, values.map((seq) => ({ @@ -106,57 +160,61 @@ describe('Agenetes conversation stores on SQLite', () => { ); } }; - const read = () => + const read = async () => kind === 'events' - ? conversationEventLogStore.readRecords(namespace, THREAD_ID) - : conversationTurnStore.list(namespace, THREAD_ID); - replace([1, 2]); + ? await conversationEventLogStore.readRecords(namespace, THREAD_ID) + : await conversationTurnStore.list(namespace, THREAD_ID); + await replace([1, 2]); return { opened, namespace, - database: substrate.database, + substrate, replace, read, }; } it('restores the complete old log when a later replacement insert fails', async () => { - const { opened, database, replace, read } = await setupReplacement(); - const before = read(); + const { opened, substrate, replace, read } = await setupReplacement(); + const before = await read(); // Fail the second insert after the delete and first insert have run. const sequence = kind === 'events' ? 'seq' : 'seq_start'; - database.exec(` - CREATE TRIGGER reject_replacement BEFORE INSERT ON agenetes_${kind} - WHEN NEW.${sequence} = 4 - BEGIN SELECT RAISE(ABORT, 'replacement insert failed'); END; - `); - expect(() => replace([3, 4])).toThrow('replacement insert failed'); - expect(read()).toEqual(before); - expect(database.isTransaction).toBe(false); - - database.exec('DROP TRIGGER reject_replacement'); + const remove = await rejectWrite( + substrate, + 'reject_replacement', + kind, + 'INSERT', + `NEW.${sequence} = 4`, + 'replacement insert failed', + ); + await expect(replace([3, 4])).rejects.toThrow( + 'replacement insert failed', + ); + expect(await read()).toEqual(before); + await remove(); + await opened.reopen(); - expect(read()).toEqual(before); - replace([3, 4]); - expect(read()).toHaveLength(2); - expect(read()).not.toEqual(before); - replace([]); - expect(read()).toEqual([]); + expect(await read()).toEqual(before); + await replace([3, 4]); + expect(await read()).toHaveLength(2); + expect(await read()).not.toEqual(before); + await replace([]); + expect(await read()).toEqual([]); }); it('leaves the old log intact when replacement serialization fails', async () => { const { namespace, read } = await setupReplacement(); - const before = read(); + const before = await read(); const cyclic: Record = {}; cyclic.self = cyclic; - expect(() => { + await expect(async () => { if (kind === 'events') { - conversationEventLogStore.replace(namespace, THREAD_ID, [ + await conversationEventLogStore.replace(namespace, THREAD_ID, [ { seq: 3, ts: 1, kind: 'turn_start', request: null }, { seq: 4, ts: 1, event: cyclic } as unknown as EventLogRecord, ]); } else { - conversationTurnStore.replace(namespace, THREAD_ID, [ + await conversationTurnStore.replace(namespace, THREAD_ID, [ { seqStart: 3, seqEnd: 3, turn: { id: 'turn-3' } as never }, { seqStart: 4, @@ -165,8 +223,8 @@ describe('Agenetes conversation stores on SQLite', () => { } as unknown as PersistedTurn, ]); } - }).toThrow(/circular/i); - expect(read()).toEqual(before); + }).rejects.toThrow(/circular/i); + expect(await read()).toEqual(before); }); }); } @@ -239,28 +297,42 @@ describe('Agenetes conversation stores on SQLite', () => { }, state: { driverState: {} }, }; - conversationThreadStore.upsert(source, THREAD_ID, record); + await conversationThreadStore.upsert(source, THREAD_ID, record); for (let seq = 1; seq <= 2; seq++) { - conversationEventLogStore.appendTurnStart(source, THREAD_ID, null); - conversationTurnStore.append(source, THREAD_ID, { + await conversationEventLogStore.appendTurnStart( + source, + THREAD_ID, + null, + ); + await conversationTurnStore.append(source, THREAD_ID, { seqStart: seq, seqEnd: seq, turn: { request: null, transcript: [] }, }); } - const snapshot = (namespace: typeof source, threadId = THREAD_ID) => ({ - record: conversationThreadStore.get(namespace, threadId), - events: conversationEventLogStore.readRecords(namespace, threadId), - turns: conversationTurnStore.list(namespace, threadId), + const snapshot = async ( + namespace: typeof source, + threadId = THREAD_ID, + ) => ({ + record: await conversationThreadStore.get(namespace, threadId), + events: await conversationEventLogStore.readRecords( + namespace, + threadId, + ), + turns: await conversationTurnStore.list(namespace, threadId), }); - const before = snapshot(source); - conversationEventLogStore.replace( + const before = await snapshot(source); + await conversationEventLogStore.replace( target, 'unrelated-thread', before.events, ); - conversationTurnStore.replace(target, 'unrelated-thread', before.turns); - const unrelated = snapshot(target, 'unrelated-thread'); + await conversationTurnStore.replace( + target, + 'unrelated-thread', + before.turns, + ); + const unrelated = await snapshot(target, 'unrelated-thread'); const empty = { record: undefined, events: [], turns: [] }; const inst = mountAgenetes({ drivers: { @@ -279,42 +351,144 @@ describe('Agenetes conversation stores on SQLite', () => { eventLogStore: conversationEventLogStore, turnStore: conversationTurnStore, }); - const substrate = conversationTables( + const substrate = await conversationSubstrate( stage.startsWith('target') ? target : source, ); if (!substrate) throw new Error('Expected SQLite conversation tables'); - // Fail a real SQLite write at each stage, including after source deletion + // Fail a real SQL write at each stage, including after source deletion // has begun. The rehome coordinator and every storage port remain real. - substrate.database.exec(` - CREATE TRIGGER reject_rehome BEFORE ${operation} ON agenetes_${table} - WHEN ${row}.extension_id = ${substrate.extensionId} - AND ${row}.thread_id = 'thread-1' ${sequence} - BEGIN SELECT RAISE(ABORT, 'rehome write failed'); END; - `); + const remove = await rejectWrite( + substrate, + 'reject_rehome', + table, + operation, + `${row}.extension_id = ${substrate.extensionId} AND ${row}.thread_id = 'thread-1' ${sequence}`, + 'rehome write failed', + ); const targetSpec = { ...record.spec, namespace: target }; - expect(() => - inst.rehome({ namespace: source, threadId: THREAD_ID }, targetSpec), - ).toThrow('rehome write failed'); - expect(snapshot(source)).toEqual(before); - expect(snapshot(target)).toEqual(empty); - expect(snapshot(target, 'unrelated-thread')).toEqual(unrelated); - expect(substrate.database.isTransaction).toBe(false); - substrate.database.exec('DROP TRIGGER reject_rehome'); + await expect( + async () => + await inst.rehome( + { namespace: source, threadId: THREAD_ID }, + targetSpec, + ), + ).rejects.toThrow('rehome write failed'); + expect(await snapshot(source)).toEqual(before); + expect(await snapshot(target)).toEqual(empty); + expect(await snapshot(target, 'unrelated-thread')).toEqual(unrelated); + await remove(); await opened.reopen(); - expect(snapshot(source)).toEqual(before); - expect(snapshot(target)).toEqual(empty); - inst.rehome({ namespace: source, threadId: THREAD_ID }, targetSpec); + expect(await snapshot(source)).toEqual(before); + expect(await snapshot(target)).toEqual(empty); + await inst.rehome({ namespace: source, threadId: THREAD_ID }, targetSpec); await opened.reopen(); - expect(snapshot(source)).toEqual(empty); - expect(snapshot(target)).toEqual({ + expect(await snapshot(source)).toEqual(empty); + expect(await snapshot(target)).toEqual({ ...before, record: { ...record, spec: targetSpec }, }); - expect(snapshot(target, 'unrelated-thread')).toEqual(unrelated); + expect(await snapshot(target, 'unrelated-thread')).toEqual(unrelated); }, ); + it('runs a mounted agent, publishes a durable tail, and recovers folded history after restart', async () => { + const opened = await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + const frames = [ + { type: 'text_delta' as const, data: { content: 'persisted answer' } }, + { type: 'end' as const, data: {} }, + ]; + const instance = mountAgenetes({ + drivers: { + test: defineDriver({ + schemaVersion: 1, + workloadTypes: ['Deployment'], + specSchema: z.object({}), + stateSchema: z.object({}), + initialState: () => ({}), + create: () => + ({ + async *run() { + for (const frame of frames) yield frame; + }, + close() {}, + }) as unknown as AgentHandle, + }), + }, + threadStore: conversationThreadStore, + eventLogStore: conversationEventLogStore, + turnStore: conversationTurnStore, + }); + const handle = await instance.create({ + kind: 'test', + workloadType: 'Deployment', + threadId: THREAD_ID, + namespace, + spec: {}, + }); + const tail = instance.tail(namespace, THREAD_ID)[Symbol.asyncIterator](); + const first = tail.next(); + const received = []; + for await (const frame of handle.run( + { type: 'user_text', content: 'question' } as never, + {} as never, + )) + received.push(frame); + expect(received).toEqual(frames); + expect((await first).value).toEqual(frames[0]); + expect((await tail.next()).value).toEqual(frames[1]); + await tail.return?.(); + const history = await instance.history(namespace, THREAD_ID); + expect(history.turns).toHaveLength(1); + expect(history.turns[0]).toMatchObject({ + request: { type: 'user_text', content: 'question' }, + }); + expect(await instance.logMetadata(namespace, THREAD_ID)).toEqual({ + eventCount: 3, + turnCount: 1, + }); + await instance.close(THREAD_ID); + await opened.reopen(); + expect( + await instance.history(canvasAcpNamespace(CANVAS_ID), THREAD_ID), + ).toEqual(history); + }); + + it('persists agent-owned documents and concurrent log appends through restart', async () => { + const opened = await openWithSpace(); + const substrate = await space(CANVAS_ID).extension('test.documents'); + if (!substrate || substrate.kind === 'disk') + throw new Error('Expected SQL extension'); + await writeSubstrateDocument(substrate, 'state', { cursor: 42 }); + expect(await readSubstrateDocument(substrate, 'state')).toEqual({ + cursor: 42, + }); + await Promise.all( + ['a', 'b', 'c'].map((value) => + appendSubstrateLog(substrate, 'prompt', '.log', value), + ), + ); + const rows = + substrate.kind === 'postgres' + ? ( + await substrate.database.query( + 'SELECT body FROM extension_documents WHERE extension_id=$1 AND name=$2', + [substrate.extensionId, 'prompt.log'], + ) + ).rows + : substrate.database + .prepare( + 'SELECT body FROM extension_documents WHERE extension_id=? AND name=?', + ) + .all(substrate.extensionId, 'prompt.log'); + expect(String(rows[0]?.body).split('').sort().join('')).toBe('abc'); + await opened.reopen(); + const fresh = await space(CANVAS_ID).extension('test.documents'); + if (!fresh) throw new Error('Missing extension'); + expect(await readSubstrateDocument(fresh, 'state')).toEqual({ cursor: 42 }); + }); + it('keeps a Space with no directory out of the file stores', async () => { await openWithSpace(); const namespace = canvasAcpNamespace(CANVAS_ID); @@ -329,45 +503,51 @@ describe('Agenetes conversation stores on SQLite', () => { await openWithSpace(); const namespace = canvasAcpNamespace(CANVAS_ID); - conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); - expect(conversationThreadStore.get(namespace, THREAD_ID)).toEqual( + await conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + expect(await conversationThreadStore.get(namespace, THREAD_ID)).toEqual( threadRecord(), ); - expect(conversationThreadStore.list(namespace)).toHaveLength(1); + expect(await conversationThreadStore.list(namespace)).toHaveLength(1); - const start = conversationEventLogStore.appendTurnStart( + const start = await conversationEventLogStore.appendTurnStart( namespace, THREAD_ID, null, ); expect(start).toMatchObject({ seq: 1, kind: 'turn_start', request: null }); - const appended = conversationEventLogStore.append(namespace, THREAD_ID, { - type: 'text', - text: 'hello', - } as never); + const appended = await conversationEventLogStore.append( + namespace, + THREAD_ID, + { + type: 'text', + text: 'hello', + } as never, + ); expect(appended.seq).toBe(2); - expect(conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe(2); + expect(await conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe( + 2, + ); // `read` is the streamed frames only; `readRecords` includes the internal // turn boundary. - expect(conversationEventLogStore.read(namespace, THREAD_ID)).toHaveLength( - 1, - ); expect( - conversationEventLogStore.readRecords(namespace, THREAD_ID), + await conversationEventLogStore.read(namespace, THREAD_ID), + ).toHaveLength(1); + expect( + await conversationEventLogStore.readRecords(namespace, THREAD_ID), ).toHaveLength(2); expect( - conversationEventLogStore.read(namespace, THREAD_ID, 2), + await conversationEventLogStore.read(namespace, THREAD_ID, 2), ).toHaveLength(0); - conversationTurnStore.append(namespace, THREAD_ID, { + await conversationTurnStore.append(namespace, THREAD_ID, { turn: { id: 'turn-1' } as never, seqStart: 1, seqEnd: 2, }); - expect(conversationTurnStore.count(namespace, THREAD_ID)).toBe(1); - expect(conversationTurnStore.fence(namespace, THREAD_ID)).toBe(2); - expect(conversationTurnStore.list(namespace, THREAD_ID)).toEqual([ + expect(await conversationTurnStore.count(namespace, THREAD_ID)).toBe(1); + expect(await conversationTurnStore.fence(namespace, THREAD_ID)).toBe(2); + expect(await conversationTurnStore.list(namespace, THREAD_ID)).toEqual([ { turn: { id: 'turn-1' }, seqStart: 1, seqEnd: 2 }, ]); }); @@ -380,27 +560,29 @@ describe('Agenetes conversation stores on SQLite', () => { .create({ canvasId: other, title: 'Other Space' }); if (!created.ok) throw new Error('Expected to create the second Space'); - conversationThreadStore.upsert( + await conversationThreadStore.upsert( canvasAcpNamespace(CANVAS_ID), THREAD_ID, threadRecord(), ); expect( - conversationThreadStore.get(canvasAcpNamespace(other), THREAD_ID), + await conversationThreadStore.get(canvasAcpNamespace(other), THREAD_ID), ).toBeUndefined(); - expect(conversationThreadStore.list(canvasAcpNamespace(other))).toEqual([]); + expect( + await conversationThreadStore.list(canvasAcpNamespace(other)), + ).toEqual([]); }); it('destroys a conversation with the Space that held it', async () => { await openWithSpace(); const namespace = canvasAcpNamespace(CANVAS_ID); - conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); - conversationEventLogStore.append(namespace, THREAD_ID, { + await conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + await conversationEventLogStore.append(namespace, THREAD_ID, { type: 'text', text: 'hello', } as never); - conversationTurnStore.append(namespace, THREAD_ID, { + await conversationTurnStore.append(namespace, THREAD_ID, { turn: { id: 'turn-1' } as never, seqStart: 1, seqEnd: 1, @@ -413,21 +595,25 @@ describe('Agenetes conversation stores on SQLite', () => { // The Space is gone, so there is no substrate to answer from — which is // the port's rule, and is also what the foreign-key cascade leaves behind. - expect(conversationThreadStore.get(namespace, THREAD_ID)).toBeUndefined(); - expect(conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe(0); - expect(conversationTurnStore.count(namespace, THREAD_ID)).toBe(0); + expect( + await conversationThreadStore.get(namespace, THREAD_ID), + ).toBeUndefined(); + expect(await conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe( + 0, + ); + expect(await conversationTurnStore.count(namespace, THREAD_ID)).toBe(0); }); it('survives a restart', async () => { const opened = await openWithSpace(); const namespace = canvasAcpNamespace(CANVAS_ID); - conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); - conversationEventLogStore.appendTurnStart(namespace, THREAD_ID, null); - conversationEventLogStore.append(namespace, THREAD_ID, { + await conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + await conversationEventLogStore.appendTurnStart(namespace, THREAD_ID, null); + await conversationEventLogStore.append(namespace, THREAD_ID, { type: 'text', text: 'hello', } as never); - conversationTurnStore.append(namespace, THREAD_ID, { + await conversationTurnStore.append(namespace, THREAD_ID, { turn: { id: 'turn-1' } as never, seqStart: 1, seqEnd: 2, @@ -438,29 +624,33 @@ describe('Agenetes conversation stores on SQLite', () => { // Same namespace, new connection: this is the whole reason these stores // exist rather than the in-memory defaults. const after = canvasAcpNamespace(CANVAS_ID); - expect(conversationThreadStore.get(after, THREAD_ID)).toEqual( + expect(await conversationThreadStore.get(after, THREAD_ID)).toEqual( threadRecord(), ); - expect(conversationEventLogStore.maxSeq(after, THREAD_ID)).toBe(2); + expect(await conversationEventLogStore.maxSeq(after, THREAD_ID)).toBe(2); expect( - conversationEventLogStore.readRecords(after, THREAD_ID), + await conversationEventLogStore.readRecords(after, THREAD_ID), ).toHaveLength(2); - expect(conversationTurnStore.fence(after, THREAD_ID)).toBe(2); + expect(await conversationTurnStore.fence(after, THREAD_ID)).toBe(2); }); it('reports an unnamed namespace as having no durable place', async () => { await openWithSpace(); const anonymous = canvasAcpNamespace(''); + await conversationThreadStore.delete(anonymous, THREAD_ID); // Agenetes's own rule: a namespace with no name is non-persistent. It must // not fall through to some other Space's tables. - expect(conversationThreadStore.list(anonymous)).toEqual([]); - conversationThreadStore.upsert(anonymous, THREAD_ID, threadRecord()); - expect(conversationThreadStore.get(anonymous, THREAD_ID)).toEqual( + expect(await conversationThreadStore.list(anonymous)).toEqual([]); + await conversationThreadStore.upsert(anonymous, THREAD_ID, threadRecord()); + expect(await conversationThreadStore.get(anonymous, THREAD_ID)).toEqual( threadRecord(), ); expect( - conversationThreadStore.get(canvasAcpNamespace(CANVAS_ID), THREAD_ID), + await conversationThreadStore.get( + canvasAcpNamespace(CANVAS_ID), + THREAD_ID, + ), ).toBeUndefined(); }); }); diff --git a/apps/server/src/modules/agent/agent-thread.service.ts b/apps/server/src/modules/agent/agent-thread.service.ts index d3f981983..33c4426cb 100644 --- a/apps/server/src/modules/agent/agent-thread.service.ts +++ b/apps/server/src/modules/agent/agent-thread.service.ts @@ -50,11 +50,16 @@ interface AgentThreadServiceDependencies { resolvePersistedExternalBinding: ( canvasId: string, threadId: string, - ) => Extract | null; + ) => + | Extract + | null + | Promise | null>; resolvePersistedSpacePrompt: ( canvasId: string, threadId: string, - ) => { realised: boolean; markdown?: string }; + ) => + | { realised: boolean; markdown?: string } + | Promise<{ realised: boolean; markdown?: string }>; collectSpacePrompt: ( canvasId: string, targetAgentNodeId: string, @@ -69,7 +74,7 @@ interface AgentThreadServiceDependencies { failLifecycle: typeof agentNodeLifecycle.error; runExternal: typeof runAcpAgent; runInternal: typeof runAgent; - closeHandle: (threadId: string) => void; + closeHandle: (threadId: string) => void | Promise; } export function externalBindingFromWorkloadSpec( @@ -101,13 +106,19 @@ const DEFAULT_DEPENDENCIES: AgentThreadServiceDependencies = { agentThreadResolver.resolveAgentNode(canvasId, threadId), resolveFixedAgentNode: (canvasId, threadId) => agentThreadResolver.resolveFixedAgentNode(canvasId, threadId), - resolvePersistedExternalBinding: (canvasId, threadId) => { - const record = agenetes.record(canvasAcpNamespace(canvasId), threadId); + resolvePersistedExternalBinding: async (canvasId, threadId) => { + const record = await agenetes.record( + canvasAcpNamespace(canvasId), + threadId, + ); if (!record || record.spec.kind !== EXTERNAL_DRIVER_KIND) return null; return externalBindingFromWorkloadSpec(record.spec.spec); }, - resolvePersistedSpacePrompt: (canvasId, threadId) => { - const record = agenetes.record(canvasAcpNamespace(canvasId), threadId); + resolvePersistedSpacePrompt: async (canvasId, threadId) => { + const record = await agenetes.record( + canvasAcpNamespace(canvasId), + threadId, + ); if (!record) return { realised: false }; const markdown = spacePromptFromWorkloadSpec(record.spec.spec); return markdown ? { realised: true, markdown } : { realised: true }; @@ -121,7 +132,7 @@ const DEFAULT_DEPENDENCIES: AgentThreadServiceDependencies = { failLifecycle: agentNodeLifecycle.error.bind(agentNodeLifecycle), runExternal: runAcpAgent, runInternal: runAgent, - closeHandle: (threadId) => agenetes.close(threadId), + closeHandle: async (threadId) => await agenetes.close(threadId), }; export class AgentThreadBusyError extends Error { @@ -229,7 +240,7 @@ export class AgentThreadService { ? { binding: fixedTarget.agentBinding, fixedTarget } : null; } - const binding = this.dependencies.resolvePersistedExternalBinding( + const binding = await this.dependencies.resolvePersistedExternalBinding( canvasId, threadId, ); @@ -261,7 +272,7 @@ export class AgentThreadService { : options.agentTarget; const persistedExternalBinding = !fixedTarget && options.canvasId - ? this.dependencies.resolvePersistedExternalBinding( + ? await this.dependencies.resolvePersistedExternalBinding( options.canvasId, options.threadId, ) @@ -308,7 +319,7 @@ export class AgentThreadService { let spacePrompt: string | undefined; try { if (agentTarget && options.canvasId && binding.kind !== 'external') { - const persisted = this.dependencies.resolvePersistedSpacePrompt( + const persisted = await this.dependencies.resolvePersistedSpacePrompt( options.canvasId, options.threadId, ); @@ -471,7 +482,7 @@ export class AgentThreadService { } } - private createDispatchStream( + private async *createDispatchStream( options: EffectiveAgentThreadInvocationOptions, binding: AgentBinding, fixedTarget: FixedAgentNodeTarget | null, @@ -483,7 +494,7 @@ export class AgentThreadService { `External thread ${options.threadId} was not canonically realized`, ); } - return this.dependencies.runExternal({ + return yield* this.dependencies.runExternal({ handle: options.externalRealization.handle, binding, threadId: options.threadId, @@ -500,10 +511,10 @@ export class AgentThreadService { const skillDispatch = planSkillDispatch(options.envelope.skills.resolved); if (skillDispatch.closeLiveHandle) { - this.dependencies.closeHandle(options.threadId); + await this.dependencies.closeHandle(options.threadId); } const runsSkillAuthoring = skillDispatch.closeLiveHandle; - return this.dependencies.runInternal({ + return yield* this.dependencies.runInternal({ scope: options.mode, workloadType: skillDispatch.workloadType, modelRole: skillDispatch.modelRole, diff --git a/apps/server/src/modules/agent/agent.route.ts b/apps/server/src/modules/agent/agent.route.ts index 5fea14ef8..858bc770a 100644 --- a/apps/server/src/modules/agent/agent.route.ts +++ b/apps/server/src/modules/agent/agent.route.ts @@ -81,7 +81,7 @@ async function dispatchBuiltinControl( ): Promise< { ok: true } | { ok: false; status: number; message: string; code: string } > { - const record = agenetes.record(namespace, threadId); + const record = await agenetes.record(namespace, threadId); if (!record) { return { ok: false, @@ -98,7 +98,7 @@ async function dispatchBuiltinControl( code: 'not_builtin', }; } - const handle = agenetes.get(threadId) ?? agenetes.create(record.spec); + const handle = agenetes.get(threadId) ?? (await agenetes.create(record.spec)); const ack = await handle.control(msg); if (!ack.ok) { return { @@ -112,11 +112,11 @@ async function dispatchBuiltinControl( } /** Read a built-in thread's per-thread selection from its durable record. */ -function readBuiltinThreadSettings( +async function readBuiltinThreadSettings( namespace: Namespace, threadId: string, -): ChatThreadSettingsResponse { - const driverState = (agenetes.record(namespace, threadId)?.state +): Promise { + const driverState = ((await agenetes.record(namespace, threadId))?.state ?.driverState ?? {}) as { modelId?: unknown; reasoningEffort?: unknown }; return { modelId: @@ -160,7 +160,7 @@ const agentRoutes: FastifyPluginAsync = async ( if (agentThreadService.isActive(threadId, canvasId)) { await agentThreadService.waitForTurnStart(threadId, canvasId); } - const { turns } = agenetes.history(namespace, threadId, { + const { turns } = await agenetes.history(namespace, threadId, { withTail: true, }); if (turns.length === 0) { @@ -169,7 +169,7 @@ const agentRoutes: FastifyPluginAsync = async ( } const messages: ChatHistoryItem[] = []; - const record = agenetes.record(namespace, threadId); + const record = await agenetes.record(namespace, threadId); const isInternalThread = (record?.spec as { kind?: unknown } | undefined)?.kind === 'internal'; buildHistoryFromTurns(turns, messages, { @@ -454,7 +454,7 @@ const agentRoutes: FastifyPluginAsync = async ( /* keep fallback */ } - const { turns } = agenetes.history( + const { turns } = await agenetes.history( canvasAcpNamespace(canvasId ?? ''), threadId, ); @@ -567,7 +567,7 @@ const agentRoutes: FastifyPluginAsync = async ( // Read lightweight L2 log metadata only to number the optional debug // prompt dump. Recovery history flows from Agenetes into the selected // driver through AgentCreateContext; the host does not load or replay it. - const { turnCount } = agenetes.logMetadata( + const { turnCount } = await agenetes.logMetadata( canvasAcpNamespace(canvasId ?? ''), resolvedThreadId, ); diff --git a/apps/server/src/modules/agent/agent.service.ts b/apps/server/src/modules/agent/agent.service.ts index a21330352..e1c8ba121 100644 --- a/apps/server/src/modules/agent/agent.service.ts +++ b/apps/server/src/modules/agent/agent.service.ts @@ -262,7 +262,7 @@ export async function* runAgent( : undefined; const durableRecord = workloadType === 'Deployment' - ? agenetes.record(namespace, deploymentThreadId) + ? await agenetes.record(namespace, deploymentThreadId) : undefined; const spec: BuiltinWorkloadSpec = buildHuabuPiWorkloadSpec({ kind: INTERNAL_DRIVER_KIND, @@ -284,7 +284,7 @@ export async function* runAgent( // Static DriverMap construction guarantees that `internal` is the // pi-backed handle. Deployments get-or-create by `threadId`; Jobs mint a // fresh handle. - const handle = agenetes.create(spec) as BuiltinHandle; + const handle = (await agenetes.create(spec)) as BuiltinHandle; // Apply any per-thread capability selection carried with this turn — a // model / reasoning effort the client picked (e.g. before the thread's diff --git a/apps/server/src/modules/canvas/canvas-search.ts b/apps/server/src/modules/canvas/canvas-search.ts index 3be81d2c1..6fc254433 100644 --- a/apps/server/src/modules/canvas/canvas-search.ts +++ b/apps/server/src/modules/canvas/canvas-search.ts @@ -244,16 +244,16 @@ function buildThreadHaystack(turns: readonly AgentTurn[]): string { * limit before invoking. No-op when the node owns no thread or the * thread is empty. */ -function scanNodeConversation( +async function scanNodeConversation( node: SearchableNode, canvasId: string, label: string | null, needleLower: string, needleLen: number, tryEmit: (match: CanvasSearchMatch) => boolean, -): void { +): Promise { if (!node.threadId) return; - const { turns } = agenetes.history( + const { turns } = await agenetes.history( canvasAcpNamespace(canvasId), node.threadId, ); @@ -684,7 +684,7 @@ export async function searchCanvas( } const content = contentByNodeId.get(node.id); const label = content?.label ?? null; - scanNodeConversation( + await scanNodeConversation( node, handle.canvasId, label, diff --git a/apps/server/src/modules/canvas/space-move.service.ts b/apps/server/src/modules/canvas/space-move.service.ts index d56e36df1..7dbcd439b 100644 --- a/apps/server/src/modules/canvas/space-move.service.ts +++ b/apps/server/src/modules/canvas/space-move.service.ts @@ -306,7 +306,7 @@ export async function moveCanvasSelection( ); } const namespace = canvasAcpNamespace(sourceCanvasId); - const record = agenetes.record(namespace, threadId); + const record = await agenetes.record(namespace, threadId); if (!record) { throw new SpaceMoveError( 'MOVE_AGENT_HISTORY_INVALID', @@ -362,7 +362,7 @@ export async function moveCanvasSelection( let sourceWrite: ExecuteOnServerOutput | undefined; try { for (const move of threadMoves) { - agenetes.rehome( + await agenetes.rehome( { namespace: canvasAcpNamespace(sourceCanvasId), threadId: move.threadId, @@ -448,7 +448,7 @@ export async function moveCanvasSelection( }); } for (const move of completedThreads.reverse()) { - agenetes.rehome( + await agenetes.rehome( { namespace: canvasAcpNamespace(destinationCanvasId), threadId: move.threadId, diff --git a/apps/server/src/modules/remote_fs/interactive-view.rfs.test.ts b/apps/server/src/modules/remote_fs/interactive-view.rfs.test.ts index 55c26b3dc..b9242dde2 100644 --- a/apps/server/src/modules/remote_fs/interactive-view.rfs.test.ts +++ b/apps/server/src/modules/remote_fs/interactive-view.rfs.test.ts @@ -33,7 +33,7 @@ async function buildApp() { return app; } -function seedCanvas() { +async function seedCanvas() { getCanvasStore('c1').write({ canvasId: 'c1', title: null, @@ -67,7 +67,7 @@ function seedCanvas() { recipe: null, }, }; - agenetes.create(ownerSpec); + await agenetes.create(ownerSpec); } const state = { @@ -83,15 +83,15 @@ const state = { value: { codebasePath: '', worktreeRoot: '' }, } as const; -beforeEach(() => { +beforeEach(async () => { workspace = mkdtempSync(join(tmpdir(), 'huabu-interactive-view-')); resetStorageCache(); setWorkspacePath(workspace); - seedCanvas(); + await seedCanvas(); }); -afterEach(() => { - agenetes.close('thread-owner'); +afterEach(async () => { + await agenetes.close('thread-owner'); resetStorageCache(); rmSync(workspace, { recursive: true, force: true }); }); diff --git a/apps/server/src/modules/storage/backends/azure/product.remote.test.ts b/apps/server/src/modules/storage/backends/azure/product.remote.test.ts new file mode 100644 index 000000000..1d110bf9c --- /dev/null +++ b/apps/server/src/modules/storage/backends/azure/product.remote.test.ts @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +// Same product journey and assertions for every selected structured/blob pair. +import '../../product-boundary.test.js'; diff --git a/apps/server/src/modules/storage/backends/postgres/conversations.remote.test.ts b/apps/server/src/modules/storage/backends/postgres/conversations.remote.test.ts new file mode 100644 index 000000000..a5ceb70ec --- /dev/null +++ b/apps/server/src/modules/storage/backends/postgres/conversations.remote.test.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import '../../../agent/agenetes/sqlite-stores.test.js'; diff --git a/apps/server/src/modules/storage/module-boundaries.test.ts b/apps/server/src/modules/storage/module-boundaries.test.ts index 05fdd322f..4e567e174 100644 --- a/apps/server/src/modules/storage/module-boundaries.test.ts +++ b/apps/server/src/modules/storage/module-boundaries.test.ts @@ -226,7 +226,9 @@ describe('storage dependency direction', () => { // Tests construct adapters directly — that is how an adapter gets // exercised. The rule is about production source: one place decides // which backend the process runs. - .filter((f) => !f.endsWith('.test.ts')) + .filter( + (f) => !f.endsWith('.test.ts') && f !== 'modules/storage/testing.ts', + ) .filter((file) => specifiersOf(file).some((spec) => { const target = resolveSpecifier(file, spec); diff --git a/apps/server/src/modules/storage/profile.test.ts b/apps/server/src/modules/storage/profile.test.ts index bd388d8a1..37555238a 100644 --- a/apps/server/src/modules/storage/profile.test.ts +++ b/apps/server/src/modules/storage/profile.test.ts @@ -58,13 +58,13 @@ describe('validateStorageProfile', () => { // A kind can be a known member of the target family while having no // adapter yet. That must fail at startup, not on first use. - it('rejects a known but unimplemented structured backend', () => { + it('accepts Postgres records beside disk blobs', () => { expect(() => validateStorageProfile({ structured: { kind: 'postgres' }, blobs: { kind: 'disk' }, }), - ).toThrow(/not implemented yet.*disk, sqlite/s); + ).not.toThrow(); }); // Fewer features is a stated limitation, not a misconfiguration: a profile @@ -80,13 +80,13 @@ describe('validateStorageProfile', () => { ).not.toThrow(); }); - it('rejects a known but unimplemented blob backend', () => { + it('accepts Azure blobs with Disk records', () => { expect(() => validateStorageProfile({ structured: { kind: 'disk' }, blobs: { kind: 'azure' }, }), - ).toThrow(/not implemented yet.*disk/s); + ).not.toThrow(); }); }); @@ -106,6 +106,7 @@ describe('requiresExplicitInit', () => { it.each([ { structured: { kind: 'postgres' }, blobs: { kind: 'disk' } }, + { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, ] as const)('requires an awaited init for %j', (profile) => { expect(requiresExplicitInit(profile)).toBe(true); diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index 4eca6a547..a304b8573 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -15,8 +15,7 @@ /** * Structured backend families a profile may name. * - * Wider than the port's `StructuredBackendKind`, which names only what an - * adapter exists for. Keeping the two apart is what lets a + * Kept distinct from the port's `StructuredBackendKind` so a future * configured-but-unwritten backend fail with "not implemented yet" instead of * "not a known backend", without the port advertising adapters that do not * exist. @@ -26,8 +25,8 @@ export type RequestedStructuredKind = 'disk' | 'sqlite' | 'postgres'; /** * Blob backend families a profile may name. * - * Wider than the port's {@link BlobBackendKind} for the same reason - * {@link RequestedStructuredKind} is wider than the structured one. + * Kept distinct from the port's {@link BlobBackendKind} for the same reason + * {@link RequestedStructuredKind} is separate from the structured one. * * Every member is a file system. Bytes are files wherever they live — a local * directory today, an object store later — and never rows in the structured @@ -53,8 +52,9 @@ export interface StorageProfile { const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ 'disk', 'sqlite', + 'postgres', ]; -const AVAILABLE_BLOBS: readonly RequestedBlobKind[] = ['disk']; +const AVAILABLE_BLOBS: readonly RequestedBlobKind[] = ['disk', 'azure']; const STRUCTURED_KINDS: readonly RequestedStructuredKind[] = [ 'disk', @@ -144,12 +144,14 @@ export function validateStorageProfile(profile: StorageProfile): void { * callers unopened. Keeping the list here, next to the other backend facts, * means adding an adapter forces a decision about it. * - * Only the structured axis appears: every blob backend is a file system, and - * a file system has no connection to open. + * Azure additionally requires awaited container validation before scopes are used. */ const LAZY_SAFE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; /** Whether this profile may only be built through an awaited `initStorage()`. */ export function requiresExplicitInit(profile: StorageProfile): boolean { - return !LAZY_SAFE_STRUCTURED.includes(profile.structured.kind); + return ( + !LAZY_SAFE_STRUCTURED.includes(profile.structured.kind) || + profile.blobs.kind !== 'disk' + ); } diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index e98a06247..08cbfe4d7 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -31,6 +31,7 @@ import { getWorkspaceHandle, getWorkspaceKey, } from '../workspace.js'; +import { AzureBlobStore } from './backends/azure/blob-store.js'; import { DiskBlobStore } from './backends/disk/blob-store.js'; import { getWorldCanvasId as diskWorldCanvasId } from './backends/disk/canvas-dirs.js'; import { @@ -42,6 +43,9 @@ import { stageDiskSpaceImport } from './backends/disk/space-import.js'; import { diskSpaceTree } from './backends/disk/space-tree.js'; import { DiskStructuredStore } from './backends/disk/structured-store.js'; import { DiskWorkspaceRepository } from './backends/disk/workspace-repository.js'; +import { PostgresStoreContext } from './backends/postgres/database.js'; +import { PostgresStructuredStore } from './backends/postgres/structured-store.js'; +import { PostgresWorkspaceRepository } from './backends/postgres/workspace-repository.js'; import { SqliteStoreContext, sqliteDatabasePath, @@ -284,6 +288,7 @@ function composeSpace(storage: Storage, canvasId: string): Space { * (see `capabilities.ts`). */ function buildBlobStore(profile: StorageProfile): BlobStore { + if (profile.blobs.kind === 'azure') return AzureBlobStore.fromEnvironment(); if (profile.blobs.kind !== 'disk') { // Unreachable: validateStorageProfile rejects unimplemented kinds. throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); @@ -299,6 +304,8 @@ function buildStructuredStore(profile: StorageProfile): StructuredStore { return new DiskStructuredStore(); case 'sqlite': return new SqliteStructuredStore(sqliteConnection()); + case 'postgres': + return new PostgresStructuredStore(postgresConnection()); default: throw new Error( `Unsupported structured backend: ${profile.structured.kind}`, @@ -349,6 +356,7 @@ export function createStorage(profile: StorageProfile): Storage { let current: Storage | null = null; let workspaces: WorkspaceRepository | null = null; let sqlite: SqliteStoreContext | null = null; +let postgres: PostgresStoreContext | null = null; let activeWorldCanvasId: string | null = null; let spaceCreateTail: Promise = Promise.resolve(); @@ -360,6 +368,15 @@ let spaceCreateTail: Promise = Promise.resolve(); * Workspace repository both borrow it, because they are one database file and * a second connection would be a second writer. */ +function postgresConnection(): PostgresStoreContext { + if (postgres) return postgres; + const connectionString = process.env['HUABU_POSTGRES_URL']; + if (!connectionString) + throw new StorageProfileError('Postgres requires HUABU_POSTGRES_URL'); + postgres = new PostgresStoreContext({ connectionString }); + return postgres; +} + function sqliteConnection(): SqliteStoreContext { if (sqlite) return sqlite; const context = new SqliteStoreContext(sqliteDatabasePath()); @@ -390,9 +407,11 @@ export function getWorkspaceRepository(): WorkspaceRepository { if (workspaces) return workspaces; const profile = activeProfile(); workspaces = - profile.structured.kind === 'sqlite' - ? new SqliteWorkspaceRepository(sqliteConnection()) - : new DiskWorkspaceRepository(workspaceRegistryPath()); + profile.structured.kind === 'postgres' + ? new PostgresWorkspaceRepository(postgresConnection()) + : profile.structured.kind === 'sqlite' + ? new SqliteWorkspaceRepository(sqliteConnection()) + : new DiskWorkspaceRepository(workspaceRegistryPath()); return workspaces; } @@ -500,7 +519,10 @@ export function adoptWorkspaceDirectory( */ export function createNamedWorkspace(name: string): Promise { const repository = getWorkspaceRepository(); - if (!(repository instanceof SqliteWorkspaceRepository)) { + if ( + !(repository instanceof SqliteWorkspaceRepository) && + !(repository instanceof PostgresWorkspaceRepository) + ) { throw new StorageProfileError( `The "${activeProfile().structured.kind}" structured backend keeps ` + 'Workspaces as directories, so a Workspace is created by adopting a ' + @@ -583,11 +605,21 @@ export async function initStorage( // memoized from the environment before an explicit profile was chosen would // answer for the wrong backend. workspaces = null; - const storage = createStorage(profile); - await Promise.all([storage.structured.init(), storage.blobs.init()]); - current = storage; - await ensureActiveWorkspace(profile); - return storage; + try { + const storage = createStorage(profile); + current = storage; + const initialized = await Promise.allSettled([ + storage.structured.init(), + storage.blobs.init(), + ]); + const failure = initialized.find((result) => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + await ensureActiveWorkspace(profile); + return storage; + } catch (error) { + await closeStorage(); + throw error; + } } /** @@ -600,13 +632,21 @@ export async function initStorage( * by a previous call — is left alone. */ async function ensureActiveWorkspace(profile: StorageProfile): Promise { - if (profile.structured.kind !== 'sqlite') return; + if (profile.structured.kind === 'disk') return; const repository = getWorkspaceRepository(); - if (!(repository instanceof SqliteWorkspaceRepository)) return; + if ( + !(repository instanceof SqliteWorkspaceRepository) && + !(repository instanceof PostgresWorkspaceRepository) + ) + return; // The question is whether *this connection* is pointed at a Workspace, not // whether the process remembers one. A handle left over from a previous // profile is a name without a namespace behind it. - if (sqliteConnection().activeWorkspaceId() !== null) return; + const context = + profile.structured.kind === 'postgres' + ? postgresConnection() + : sqliteConnection(); + if (context.activeWorkspaceId() !== null) return; const workspace = await repository.ensureDefault(DEFAULT_WORKSPACE_NAME); await activateWorkspace(workspace); } @@ -629,9 +669,13 @@ export async function activateWorkspace( // or the SQLite connection moves away from the current Workspace. commitWorkspaceIdentity(workspace); if (sqlite) sqlite.useWorkspace(workspace.workspaceId); + if (postgres) postgres.useWorkspace(workspace.workspaceId); activeWorldCanvasId = null; - if (workspaces instanceof SqliteWorkspaceRepository) { - workspaces.markOpened(workspace.workspaceId); + if ( + workspaces instanceof SqliteWorkspaceRepository || + workspaces instanceof PostgresWorkspaceRepository + ) { + await workspaces.markOpened(workspace.workspaceId); } // A Workspace with no World has no Portal target and no home view. On Disk // the World is written by workspace preparation; here the same step belongs @@ -684,16 +728,29 @@ export function getStorage(): Storage { export async function closeStorage(): Promise { const storage = current; const connection = sqlite; + const postgresContext = postgres; current = null; workspaces = null; sqlite = null; + postgres = null; activeWorldCanvasId = null; - if (storage) { - await Promise.all([storage.structured.close(), storage.blobs.close()]); + try { + if (storage) { + const results = await Promise.allSettled([ + storage.structured.close(), + storage.blobs.close(), + ]); + const failure = results.find((result) => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + } + } finally { + // Composition owns shared connections, including failed initialization. + try { + connection?.close(); + } finally { + await postgresContext?.close(); + } } - // The shared connection outlives either store, so closing it is this - // module's job rather than whichever adapter happens to hold it. - connection?.close(); } export function getBlobStore(): BlobStore { @@ -766,7 +823,10 @@ export async function deleteSpace( // directory those areas sat in. Sweeping the areas is the port's // contract; removing what composition placed them under is this // module's, and it is what stops a deleted Space leaving a husk behind. - if (storage.profile.structured.kind !== 'disk') { + if ( + storage.profile.structured.kind !== 'disk' && + storage.profile.blobs.kind === 'disk' + ) { await rm(detachedSpaceRoot(canvasId), { recursive: true, force: true, diff --git a/apps/server/src/modules/storage/testing.ts b/apps/server/src/modules/storage/testing.ts index 658f2a96b..a874bafdc 100644 --- a/apps/server/src/modules/storage/testing.ts +++ b/apps/server/src/modules/storage/testing.ts @@ -21,6 +21,8 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; +import { prepareAzureTestEnvironment } from './backends/azure/test-support.js'; +import { preparePostgresTestEnvironment } from './backends/postgres/test-support.js'; import { closeStorage, initStorage } from './storage.js'; import { setWorkspacePath } from '../workspace.js'; @@ -37,6 +39,35 @@ import type { Storage } from './storage.js'; export const PRODUCT_STORAGE_PROFILES: readonly StorageProfile[] = [ { structured: { kind: 'disk' }, blobs: { kind: 'disk' } }, { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, + ...(process.env['HUABU_TEST_POSTGRES_URL'] + ? [ + { + structured: { kind: 'postgres' as const }, + blobs: { kind: 'disk' as const }, + }, + ...(process.env['HUABU_TEST_AZURE_CONNECTION_STRING'] + ? [ + { + structured: { kind: 'postgres' as const }, + blobs: { kind: 'azure' as const }, + }, + ] + : []), + ] + : []), + // Dedicated service-backed test command exercises the selectable Azure profiles. + ...(process.env['HUABU_TEST_AZURE_CONNECTION_STRING'] + ? [ + { + structured: { kind: 'disk' as const }, + blobs: { kind: 'azure' as const }, + }, + { + structured: { kind: 'sqlite' as const }, + blobs: { kind: 'azure' as const }, + }, + ] + : []), ]; /** Readable name for a profile, for test titles. */ @@ -108,10 +139,41 @@ export async function mountTestWorkspace( process.env['HUABU_BLOB_ROOT'] = path.join(workspacePath, 'blobs'); } - const storage = await initStorage(profile); - // A namespace nobody has opened before has no World, and a Workspace - // without one has no home view. Every backend meets that state once. - await storage.structured.spaces().ensureWorld(); + let releasePostgres: (() => Promise) | null = null; + let releaseAzure: (() => Promise) | null = null; + const cleanup = async () => { + try { + await closeStorage(); + } finally { + try { + await releaseAzure?.(); + } finally { + try { + await releasePostgres?.(); + } finally { + restoreEnv('HUABU_SQLITE_PATH', previousSqlitePath); + restoreEnv('HUABU_BLOB_ROOT', previousBlobRoot); + rmSync(workspacePath, { recursive: true, force: true }); + } + } + } + }; + let storage: Storage; + try { + releasePostgres = + profile.structured.kind === 'postgres' + ? await preparePostgresTestEnvironment() + : null; + releaseAzure = + profile.blobs.kind === 'azure' + ? await prepareAzureTestEnvironment() + : null; + storage = await initStorage(profile); + await storage.structured.spaces().ensureWorld(); + } catch (error) { + await cleanup(); + throw error; + } return { profile, @@ -124,12 +186,7 @@ export async function mountTestWorkspace( await reopened.structured.spaces().ensureWorld(); return reopened; }, - async close(): Promise { - await closeStorage(); - restoreEnv('HUABU_SQLITE_PATH', previousSqlitePath); - restoreEnv('HUABU_BLOB_ROOT', previousBlobRoot); - rmSync(workspacePath, { recursive: true, force: true }); - }, + close: cleanup, }; } diff --git a/docs/README.md b/docs/README.md index cdc77e692..d91ad6b62 100644 --- a/docs/README.md +++ b/docs/README.md @@ -97,7 +97,7 @@ docs/ | [milkdown-custom-toolbar-plan.md](./proposals/milkdown-custom-toolbar-plan.md) | In-Progress | Huabu-owned Milkdown toolbar and semantic editor commands. | | [model-role-routing.md](./proposals/model-role-routing.md) | Proposed | Model selection by runtime role. | | [move-selected-nodes-between-spaces.md](./proposals/move-selected-nodes-between-spaces.md) | Proposed | #142 selected-node and Frame-subtree moves between Spaces with bounded compensation. | -| [multi-backend-storage.md](./proposals/multi-backend-storage.md) | Partly shipped | Phases 1–3: Blob, structured repositories, catalogue, and bounded reads. | +| [multi-backend-storage.md](./proposals/multi-backend-storage.md) | Partly shipped | Phases 1–6 implemented, including Postgres, Azure Blob, and async agent persistence. | | [note-auto-height-stable-geometry.md](./proposals/note-auto-height-stable-geometry.md) | Proposed | Revision-aware offscreen Note measurement and stable auto-height geometry. | | [space-preview-and-world-redesign.md](./proposals/space-preview-and-world-redesign.md) | In-Progress | View-only Space previews, a preview-based World, and deferred zoom-through navigation. | | [space-prompt-topology-scoping.md](./proposals/space-prompt-topology-scoping.md) | Shipped | Topology-derived global/direct-Agent targeting for Prompt Frames. | diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index c889debc1..7c177a1e9 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -59,7 +59,7 @@ Key points: - `space(canvasId)` is the one entry point to a Space. It is a composition-layer facade, not a port type: `StructuredStore` and `BlobStore` never import each other, and they are joined only where the cross-store rules already live — the blob-put precondition and the blob-first delete saga. It composes from its receiver, so substituting one axis on a `Storage` object yields Spaces built on the substitute. - A capability only one backend has hangs off that same handle, named for the backend and typed by its absence rather than stubbed to throw: `diskTree` is the Disk Space directory and is `null` on every other backend. It is not a port and does not live in `ports/`. `module-boundaries.test.ts` holds its exact production consumer census — a list that may shrink and must not grow — and asserts the barrel exposes nothing that reads as a portable path API. - A Space's bytes are reached the same way as its records: `BlobStore.space(id)` returns one member per user-visible area — `artifacts`, `guide`, `memory`, `uploads` — so the Disk paths a user sees are unchanged and retention can diverge later without moving bytes. The `guide` area is bounded by its member names rather than by a directory, because its area is the Space root: a directory scope there would let `list()` claim `space.json` and `deleteAll()` remove the Space. Rename and per-key delete remain unsupported. -- `space(canvasId).extension(namespace)` hands an owner an isolated place to keep its own per-Space state — a reserved directory on Disk, a shared connection plus a Space-owned parent row on SQLite — and nothing else. Storage validates the namespace, creates it on demand, and destroys it with the Space, which is the one operation an owner cannot perform itself; it guarantees nothing about the contents, and cannot, because it never sees them. `extension()` returns `null` for a Space that is gone, which is where the per-owner `existsSync` resurrection guards went. Memory-worker bookkeeping, the debug prompt log, and the Agenetes conversation stores are its owners; ACP session state is assigned here but moves with the Agenetes `Namespace` change. SQLite event-log and turn-log replacement pre-encodes every record, then uses a savepoint to make deletion and insertion atomic, preserving the old log if replacement fails. Because Agenetes's storage ports are synchronous and `extension()` is not, the composition root also exposes `sqliteTree` — the synchronous form of the same resolution, named for the backend that has it and `null` elsewhere, with its own single-consumer census beside `diskTree`'s. +- `space(canvasId).extension(namespace)` hands an owner an isolated place to keep its own per-Space state — a reserved directory on Disk, a shared connection plus a Space-owned parent row on SQLite — and nothing else. Storage validates the namespace, creates it on demand, and destroys it with the Space, which is the one operation an owner cannot perform itself; it guarantees nothing about the contents, and cannot, because it never sees them. `extension()` returns `null` for a Space that is gone, which is where the per-owner `existsSync` resurrection guards went. Memory-worker bookkeeping, the debug prompt log, and the Agenetes conversation stores are its owners; ACP session state is assigned here but moves with the Agenetes `Namespace` change. SQLite event-log and turn-log replacement pre-encodes every record, then uses a savepoint to make deletion and insertion atomic, preserving the old log if replacement fails. Agenetes awaits asynchronous persistence. SQLite retains `sqliteTree` internally for its synchronous store implementation; Postgres resolves the async extension substrate. Both keep conversation data in owner tables with a Space-owned cascade parent. - Features that are _about_ a filesystem are declared, not emulated. `capabilities.ts` lists Workspace folder selection, bundle export and import, reveal-in-file-manager, the built-in file tools, RFS's file plane, external-note discovery, the Workspace memory document, and user-authored skills as Disk-only; startup logs the ones the selected profile does not offer and each refusal reuses that same wording. An unavailable feature is a stated limitation and startup continues, while a profile naming an unimplemented backend stays a misconfiguration that fails fast. Fewer features is therefore not a reason to make a backend unselectable — an _undeclared_ gap is. Windows directory-handle coordination applies only when a directory operation invokes it; it is not a declared capability or a limitation of SQLite. - `closeStorage()` closes both connections on graceful Server shutdown and forgets the holder. On Disk it releases nothing a process exit would not, and it exists for the backend that will hold a pool. - `SpaceRepository.ensureWorld()` is the backend-neutral World bootstrap: it returns the established World or mints exactly one version-0 World when the namespace holds none. An _established_ World that is missing or malformed stays the integrity error `worldId()` reports, because regenerating identity there would orphan every reference to it. Disk delegates to the same idempotent primitive Workspace preparation calls, so one file keeps one writer. @@ -78,7 +78,7 @@ Key points: - Remote PDF preprocessing writes the already-fetched source bytes into the Space BlobStore as `artifact-.pdf` before structured persistence and replaces the node's remote `src` with that key. As with other artifact imports, this blob write precedes the node write operation; a later structured persistence failure may therefore leave an unreferenced blob until Space deletion, while a blob-write failure degrades to retaining the remote URL. - Events are append-only JSONL (`events.jsonl`); each line is `{ ts: number, payload: RecentAction }`. - The memory analyzer reads Space existence and at most 100 recent action events through one `SpaceHandle`. A missing Space skips the pass before reading memory files or calling the model; corrupt part data still fails the pass. Memory body/state files remain materialized workspace paths, while Agenetes-owned chat history is not part of the curator bundle. -- **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** Which store owns it depends on where the Space lives: a namespace carrying a `storage.root` (Disk) uses the file stores described below, a Space in SQLite uses the `agenetes_*` tables, and an unnamed namespace stays in memory as Agenetes intends. On Disk the canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. +- **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** Which store owns it depends on where the Space lives: a namespace carrying a `storage.root` (Disk) uses the file stores described below, a Space in SQLite or Postgres uses the `agenetes_*` tables, and an unnamed namespace stays in memory as Agenetes intends. On Disk the canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. - Durable Agenetes workload records live in `.history/threads.json` (`agenetes-v2` schema, one record per thread; written by `FileThreadStore`). The host-local `namespace.storage.root` is never persisted: reads bind each record to the current Space namespace, so a Home synchronized across computers cannot redirect storage back to another machine's absolute path. - Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`/`runs.complete`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. - Legacy chat files are one-way migrated into `chat_v2/` at workspace activation and retired to `.bak`: the oldest pi-ai `Context` `chat/.json` via `migrate-chat-threads.ts` (hop 1), then the M5.6 `chat/.turns.jsonl` / `.active.json` via `migrate-chat-turns.ts` (hop 2). If hop 1 finds both formats after an interrupted launch, it completes a strict converted prefix atomically or preserves an existing tail when the full conversion is its prefix. Divergent logs are retained rather than guessed or overwritten; hop 2 skips the paired turn log while a valid same-thread legacy Context remains or its JSON cannot be read safely, so a later activation can retry both copies without blocking unrelated migrations. The obsolete `CanvasStore` chat methods and `chatPath()` helper were removed in Phase 2; `chatDir()` remains because change-review and agent-owned files still use that directory. @@ -138,6 +138,38 @@ Notes an operator needs: These are keyed on the **profile**, both axes. Most need a Space or Workspace to be a real directory (a structured-backend property); bundle export, bundle import, the file tools, and RFS additionally need the Space's _bytes_ to be in that directory, which is the blob backend's. Each row states a requirement with one clause per axis: every clause present must hold (`and` across axes), any listed backend satisfies its own clause (`or` within one), and an absent clause requires nothing — so a blob-agnostic row never needs editing when a blob backend is added. On the hybrid profile the answer is unchanged: a file system for the bytes hands nothing back, because every row still needs the record and node documents to be files. Refusals ask `storageServes(id)`; `module-boundaries.test.ts` checks that every declared row is refused somewhere. +## 2c. Postgres and Azure profiles + +Phase 6 makes all six structured/blob pairings selectable. Configure +`HUABU_STRUCTURED_BACKEND=postgres` with `HUABU_POSTGRES_URL` for a dedicated +Huabu database/schema. Workspaces, Spaces, nodes, logs, Tasks, extension parents, +and agent conversations live in Postgres. Schema migration runs transactionally +at startup; a newer schema is rejected. Ordered writes, CAS and name allocation +use native transactions with a database advisory lock. This is still a +single-Server application topology; SQL locking does not provide distributed +admission across a blob deletion saga. + +Set `HUABU_BLOB_BACKEND=azure`, `HUABU_AZURE_STORAGE_CONNECTION_STRING`, and +`HUABU_AZURE_BLOB_CONTAINER` to use an existing private Azure container. +`HUABU_AZURE_BLOB_PREFIX` defaults to `huabu`; keys include Workspace identity, +Space id, and area. Uploads stage blocks and publish only on successful commit. +Materialization creates a temporary local file whose lease must be disposed. +Operators provision the container; deleting a Space only sweeps its scopes. +Filesystem capabilities keep their existing declarations, including the features +that require both Disk axes. Local runtime state still needs a writable data directory. + +Agenetes lifecycle and durable reads are awaited throughout the server. Thread +snapshots persist before notifications; events persist before live publication; +close drains pending state writes. Conversation rehome awaits all writes and +compensation. Memory bookkeeping and debug prompt logs use async extension +helpers on each substrate. SQLite and Postgres share codecs, validation, identity +allocation, ordered-write checks, and Task/Run transitions; each owns its dialect, +queries, migrations, and transaction mechanics. + +Run `pnpm test:storage-backends` with Docker available for disposable PostgreSQL +and Azurite adapter, conversation, and six-profile product tests. CI runs the same +command. Live Azure account verification and multi-Server coordination are separate work. + ## 3. Storage composition and ownership `apps/server/src/modules/storage/` has three layers plus its composition root: @@ -158,7 +190,7 @@ Notes an operator needs: | `index.ts` | Public exports only; application code imports here rather than reaching into an adapter. | | `canvas-store.ts`, `paths.ts`, `canvas-dirs.ts` | Deprecated forwarding shims with no logic, retained only for high-fanout compatibility imports. | -The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. The SQLite adapter instead owns one explicit database filename and connection; retained handles stay bound to that connection, and its `init`, `health`, and `close` lifecycle is exercised only by direct tests. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; SQLite uses a native transaction. +The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. The SQLite adapter instead owns one explicit database filename and connection; retained handles stay bound to that connection, and its `init`, `health`, and `close` lifecycle is exercised by adapter and mounted product tests. Postgres owns an async connection pool with the same Workspace binding. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; SQLite uses a native transaction. Canvas persistence DTOs and the write coordinator live under `modules/canvas/`; Workspace identity, durable membership, and Disk locators live under `modules/storage/`, while active-Workspace lifecycle and boot migrations remain under `modules/workspace/`; generic filesystem and Markdown codecs live under `utils/`. `canvasRoot()` validates the identifier and then verifies that the resolved Space directory remains a strict descendant of the active Workspace before any downstream Disk operation receives it. `module-boundaries.test.ts` enforces the storage dependency direction, prevents new consumers of the forwarding shims, and holds the neutrality guard: no production file outside `storage/` may import a Disk layout symbol or a legacy `CanvasStore` symbol. The check is import-level and symbol-level — a local variable that happens to be called `artifactPath` is not a violation, while importing `canvasRoot`, or reaching `getCanvasStore` through the barrel, is. Migrations are exempt because they rewrite frozen historical on-disk shapes; tests are exempt for the same reason they may name an adapter. @@ -188,7 +220,7 @@ The launch path deliberately has no compensation transaction. A launch failure l ### 3.3 Task Run completion -`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. Both structured adapters perform lookup, `running → completed`, and persistence inside one Task-snapshot mutation boundary: Disk uses the per-Canvas Task mutex and atomic file replacement, while SQLite uses an immediate transaction. HTTP and built-in-tool callers therefore share one atomic transition rather than performing a read-then-update race. +`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. All structured adapters perform lookup, `running → completed`, and persistence inside one Task-snapshot mutation boundary: Disk uses the per-Canvas Task mutex and atomic file replacement, while SQLite and Postgres use native transactions and share Task/Run transition rules. HTTP and built-in-tool callers therefore share one atomic transition rather than performing a read-then-update race. A completed Run stores immutable `completion.completedAt` and an optional trimmed caller-owned `completion.message`. The platform treats the message as untrusted text and does not interpret issue, pull-request, or URL semantics. A retry with the same normalized message is idempotent and preserves the original timestamp; a different message conflicts. A `pending` Run cannot complete, and Agent turn termination never implies Run completion. diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index d95697d0f..3863993bb 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -1,6 +1,6 @@ # Multi-Backend Storage -Status: Phases 1–5 implemented; Phase 6 adapter foundation implemented; activation follows +Status: Phases 1–5 implemented; Phase 6 implemented — Postgres, Azure, and async agent persistence Last updated: 2026-09-14 > **Scope and decision confidence.** This proposal records the two-port @@ -59,7 +59,7 @@ Last updated: 2026-09-14 > harness, **implemented**). §12 is the authoritative plan; > the decision table in §2 marks what each step has actually settled. > -> Phase 5 is specified in §12.9 and is **implemented by this branch**. +> Phase 5 is specified in §12.9 and is **merged to main in PR #92**. > `HUABU_STRUCTURED_BACKEND=sqlite` is a real profile: Workspaces, Spaces, > nodes, logs, Tasks, and agent conversations are rows in one database file > under `/storage/sqlite/`, and the deployment needs no Workspace @@ -67,7 +67,9 @@ Last updated: 2026-09-14 > are always files — SQL records beside ordinary files is the profile, not a > compromise within it. What it does **not** serve is enumerated in §12.9.4 > and declared in `storage/capabilities.ts`, which is the list an operator -> sees at startup. Postgres and Azure adapters still do not exist. +> sees at startup. Phase 6 (§12.10) adds selectable Postgres and Azure adapters, +> including asynchronous Agenetes persistence. All six axis pairings are supported +> within the existing single-Server topology and capability declarations. --- @@ -97,8 +99,8 @@ built above these ports, but its form is intentionally unresolved here. | Topic | Status | Current position | | ------------------------------------------------------ | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | -| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk and SQLite are selectable; Postgres has no adapter. | -| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations — both file systems. Only Disk exists. A structured backend never holds bytes, so the two axes share nothing and any implemented pairing is a valid deployment. | +| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk, SQLite, and Postgres are selectable (§12.10). | +| Blob backend family | **Settled direction** | Disk and Azure Blob are implemented and selectable (§12.10). A structured backend never holds bytes, so the two axes share nothing and any implemented pairing is a valid deployment. | | Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | | Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | | Concrete interface shape and async migration | **Accepted** (P4) | Blob and portable structured repositories are async. `StructuredStore` exposes catalogue/lifecycle and scoped Space handles; the structured mutations enumerated in §12.4 use those ports. Disk-only physical capabilities remain explicit blockers for selecting another profile. | @@ -153,8 +155,9 @@ external-note discovery watches `nodes/`, and export archives the entire Space directory. Therefore wrapping `CanvasStore` in a database adapter would not by itself make the application backend-neutral. -Runtime Canvas/Space persistence is Disk by default and SQLite by selection -(§12.9). Postgres and Azure Blob adapters do not yet exist. +Runtime Canvas/Space persistence defaults to Disk. SQLite (§12.9) and Postgres +(§12.10) are selectable; each structured backend composes with Disk or Azure +Blob storage. Filesystem capabilities remain explicitly declared. ## 4. Goals @@ -175,9 +178,8 @@ Runtime Canvas/Space persistence is Disk by default and SQLite by selection ## 5. Non-goals -- Selecting a production ORM, SQL query builder, Postgres driver, or final - SQLite driver. The isolated Phase 5 preview uses built-in `node:sqlite` - without making that production choice. +- Introducing an ORM or generic query builder. SQLite uses `node:sqlite`; + Postgres uses `pg`, with native transaction ownership in each adapter. - Defining the final relational schema or migration framework. - Choosing a VFS, FUSE, materialization, cache, or write-back design. - Replacing RFS or the canonical `SpaceQuery` / `CanvasCommand` contracts in @@ -228,9 +230,8 @@ move L2 persistence ownership back into `CanvasStore`. Every current Canvas structured port is asynchronous so a synchronous Disk or SQLite implementation does not constrain Postgres or another remote adapter. -The corresponding migration for Agenetes ports that are synchronous today -remains future work; a blocking compatibility facade over Postgres is not an -acceptable end state. +Phase 6 migrates Agenetes persistence and extension-document helpers to await +asynchronous stores, preserving persist-before-notify and ordered writes. **As implemented**, Phase 1 landed `StructuredStore` as a backend-selection boundary whose handle was still synchronous and filename-shaped. Phase 2 @@ -960,10 +961,9 @@ explicitly: ## 12. Migration plan -Phases 1–4.5 are implemented and merged. Phase 5 is implemented by this -isolated contract preview. Phase 6 onward keeps the provisional character of -the original outline: those entries record intended order, not approved -designs. +Phases 1–5 are implemented and merged. Phase 6 adds the remaining adapters, +async agent persistence, and full Postgres profile activation. Phase 7 onward +remains provisional: those entries record intended order, not approved designs. The current on-disk format remains readable throughout port extraction. A database adapter must not require Disk consumers to simulate tables, and the @@ -2756,27 +2756,73 @@ across a switch, and forget-without-delete. The Agenetes conversation stores have their own suite against a mounted profile, covering round-trip, isolation, restart, and destruction with the Space. -### 12.10 Phase 6 — adapter foundation and application activation - -Phase 6 is split into two reviewable steps. This foundation implements native -Postgres structured repositories and Azure Blob storage, shared SQL codecs, -validation, name allocation, and Task/Run rules, plus async extension-document -helpers. Existing Disk and SQLite behavior remains covered by its tests. -`pnpm test:storage-backends` provisions disposable PostgreSQL and Azurite -services and runs the adapter contracts, including rollback, independent -connection CAS, Workspace isolation, staged-upload failure, and same-key -upload serialization. CI runs the same command. - -This foundation does not select the new adapters for the application. The -stacked follow-up migrates Agenetes persistence and its callers to async, -activates all six structured/blob pairings, and adds conversation and product -coverage to the same harness. The earlier phase sections below and above -record their original scope; this section defines the current Phase 6 split. +### 12.10 Phase 6 — Postgres, Azure Blob, and async agent persistence — **implemented** + +Phase 6 is reviewed as two stacked changes: an independently verified adapter +foundation (shared SQL logic, native adapters, extension documents, and CI), +followed by async Agenetes persistence, application activation, and product +coverage. The foundation keeps the new profiles unselectable until the second +change is applied. + +Phase 6 implements the remaining backend families. Work is on +`feat/multi-backend-storage-phase-6`, based on `main` after Phase 5 merged. + +- **Postgres:** an asynchronous `pg` adapter for Workspaces, Space catalogue, + lifecycle, nodes, ordered record/node/delta writes, events, changes, Tasks, + and extension parent rows. Real PostgreSQL transactions preserve CAS, + revision safety, name allocation, and rejected-batch rollback. The initial + implementation serializes adapter transactions using a database advisory + lock; it does not claim multi-Server application support or a distributed + deletion fence across blob I/O. +- **Azure Blob:** streamed block staging with atomic publication, metadata, + inclusive range reads, bounded existence queries, listing, temporary-file + materialization leases, and deletion of all Space areas. Keys include a + deployment prefix, Workspace identity, Space id, and area. Operators own + container provisioning and credentials; Space deletion never deletes the + container. Same-key staging/commit uses the shared process-local keyed mutex + because an Azure commit discards competing uncommitted blocks. Independent + keys upload concurrently; distributed writer coordination remains future work. +- **Shared SQL behavior:** SQLite and Postgres share JSON encoding/decoding, + title and label allocation, validation, and Task/Run transitions. Synchronous + SQLite transactions and asynchronous Postgres transactions remain separate + owners; SQLite's immutable migration history is preserved. +- **Async agent persistence and activation:** Agenetes accepts synchronous or + asynchronous ThreadStore, EventLogStore, and TurnStore implementations. Its + create/fork/rehome/close and durable read surfaces return promises; callers + await them. Event appends persist before publication, state reports queue + before notifications, close drains pending reports, and lifecycle operations + serialize across rehome's compensation boundary. Postgres conversation + tables remain owned by Agenetes and cascade with their extension parent. + Memory bookkeeping and debug prompt documents also persist on Postgres. + All six structured/blob pairings are selectable. Postgres never falls back + to local conversation files or an in-memory named namespace. +- **Verification:** `pnpm test:storage-backends` provisions disposable PostgreSQL + 17 and Azurite containers and runs the shared contracts, database rollback, + independent-connection CAS/name-allocation, restart and Workspace isolation, + conversation rehome compensation, and all six product profiles. CI runs this + command separately from ordinary tests. Async runtime tests delay and reject + persistence to verify ordering and notification behavior. Azure coverage + includes staged-upload failure, multiblock and zero-byte uploads, ranges, + Workspace isolation, deletion, and temporary lease cleanup. Azurite evidence + does not replace a cloud-account probe. + +Configuration (credentials belong in deployment environment/secret management): + +- `HUABU_STRUCTURED_BACKEND=disk|sqlite|postgres` (default `disk`). +- `HUABU_BLOB_BACKEND=disk|azure` (default `disk`). +- Postgres requires `HUABU_POSTGRES_URL`, a `pg` connection URL. The selected + database/schema must be dedicated to Huabu and permit table, index, and + extension-owner table creation. TLS is configured through the connection URL. +- Azure requires `HUABU_AZURE_STORAGE_CONNECTION_STRING` and + `HUABU_AZURE_BLOB_CONTAINER` for an existing private container. Optional + `HUABU_AZURE_BLOB_PREFIX` defaults to `huabu`; use a distinct prefix/container + for each deployment. The connection must permit read, list, write, and delete. +- SQL + Disk blobs uses `HUABU_BLOB_ROOT` (or the existing data-directory + default). Local runtime state and temporary materializations still require + local disk even with Postgres/Azure. ### 12.11 Later phases — provisional -6. Migrate the currently synchronous Agenetes persistence ports without - changing their persist-before-notify, sequence, and fencing semantics. 7. Refactor RFS and built-in file tools only after a logical file-view contract is accepted, if that option is chosen. 8. Prototype native CLI access separately and decide between protocol-only, diff --git a/external/agenetes/README.md b/external/agenetes/README.md index 2981768fd..5d941687e 100644 --- a/external/agenetes/README.md +++ b/external/agenetes/README.md @@ -48,23 +48,23 @@ Each surface has an in-process programmatic form today, used when Agenetes is mo 每个 surface 当前都有一种进程内的程序调用形态，用于 Agenetes 被直接 mount 进 host application 的场景。下表中的 API-shaped forms 是未来跨进程或网络边界时的投影建议；它们描述的是预期的 REST/SSE 形态，而不是最终 HTTP contract。 -| Surface | Current in-process API | Suggested API-shaped form _(planned)_ | Meaning | -| ------------------- | ------------------------------------------------------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | -| Instance | `Agenetes.create(spec) -> AgentHandle` | `POST /workloads` | Realize a `WorkloadSpec`: Jobs mint a fresh handle; Deployments get-or-create the live handle by `threadId`. | -| Instance | `Agenetes.get(threadId) -> AgentHandle \| undefined` | `GET /workloads/:threadId/live` | Return the live Deployment handle when one is already running; never spawns. | -| Instance | `Agenetes.close(threadId) -> void` | `DELETE /workloads/:threadId/live` | Close and evict the live handle for a thread. | -| Agent Handle | `AgentHandle.run(submission, ctx) -> AsyncGenerator` | `POST /workloads/:threadId/runs` + stream | Run one turn, stream `AgentStreamEvent`s, and return the driver's per-turn result. | -| Agent Handle | `AgentHandle.control(msg) -> Promise` | `POST /workloads/:threadId/control` | Send an out-of-turn `ControlMsg` and receive a `ControlAck`. | -| Agent Handle | `AgentHandle.close() -> void` | `DELETE /workloads/:threadId/live` | Release this workload through the handle surface. | -| Agent Handle | `AgentHandle.capabilities -> AgentCapabilities` | `GET /workloads/:threadId/capabilities` | Read the operations and features this handle advertises. | -| Persistent Querying | `Agenetes.record(namespace, threadId) -> ThreadRecord \| undefined` | `GET /namespaces/:namespace/workloads/:threadId` | Read one durable thread record independent of handle liveness. | -| Persistent Querying | `Agenetes.records(namespace) -> ThreadRecord[]` | `GET /namespaces/:namespace/workloads` | Enumerate persisted thread records in one namespace. | -| Persistent Querying | `Agenetes.notifications(threadId) -> AsyncIterable` | `GET /workloads/:threadId/notifications` | Subscribe to persisted AgentMetadata updates. | -| Persistent Querying | `Agenetes.logMetadata(namespace, threadId) -> ThreadLogMetadata` | `GET /namespaces/:namespace/workloads/:threadId/log-metadata` | Read Tier-1 event and Tier-2 folded-turn counts without loading either log. | -| Persistent Querying | `Agenetes.history(namespace, threadId, { withTail? }) -> ThreadHistory` | `GET /namespaces/:namespace/workloads/:threadId/history?tail=1` | Read folded turns, optionally projecting the current Tier-1 tail as an incomplete turn snapshot. | -| Persistent Querying | `Agenetes.tail(namespace, threadId) -> AsyncIterable` | `GET /namespaces/:namespace/workloads/:threadId/events` | Follow the live Tier-1 event tail after the latest folded turn. | -| Configuration | `defineDriver(definition) -> MountedAgentDriver` | deployment / configuration API | Bind one driver's schema version, workload types, spec/state schemas, initial state, and typed implementation. | -| Configuration | `mountAgenetes({ drivers, ...stores }) -> Agenetes` | deployment / configuration API | Mount a complete static `kind → driver` map with instance-level persistence and recovery policy. | +| Surface | Current in-process API | Suggested API-shaped form _(planned)_ | Meaning | +| ------------------- | -------------------------------------------------------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Instance | `Agenetes.create(spec) -> Promise` | `POST /workloads` | Realize a `WorkloadSpec`: Jobs mint a fresh handle; Deployments get-or-create the live handle by `threadId`. | +| Instance | `Agenetes.get(threadId) -> AgentHandle \| undefined` | `GET /workloads/:threadId/live` | Return the live Deployment handle when one is already running; never spawns. | +| Instance | `Agenetes.close(threadId) -> Promise` | `DELETE /workloads/:threadId/live` | Close and evict the live handle for a thread. | +| Agent Handle | `AgentHandle.run(submission, ctx) -> AsyncGenerator` | `POST /workloads/:threadId/runs` + stream | Run one turn, stream `AgentStreamEvent`s, and return the driver's per-turn result. | +| Agent Handle | `AgentHandle.control(msg) -> Promise` | `POST /workloads/:threadId/control` | Send an out-of-turn `ControlMsg` and receive a `ControlAck`. | +| Agent Handle | `AgentHandle.close() -> void` | `DELETE /workloads/:threadId/live` | Release this workload through the handle surface. | +| Agent Handle | `AgentHandle.capabilities -> AgentCapabilities` | `GET /workloads/:threadId/capabilities` | Read the operations and features this handle advertises. | +| Persistent Querying | `Agenetes.record(namespace, threadId) -> Promise` | `GET /namespaces/:namespace/workloads/:threadId` | Read one durable thread record independent of handle liveness. | +| Persistent Querying | `Agenetes.records(namespace) -> Promise` | `GET /namespaces/:namespace/workloads` | Enumerate persisted thread records in one namespace. | +| Persistent Querying | `Agenetes.notifications(threadId) -> AsyncIterable` | `GET /workloads/:threadId/notifications` | Subscribe to persisted AgentMetadata updates. | +| Persistent Querying | `Agenetes.logMetadata(namespace, threadId) -> Promise` | `GET /namespaces/:namespace/workloads/:threadId/log-metadata` | Read Tier-1 event and Tier-2 folded-turn counts without loading either log. | +| Persistent Querying | `Agenetes.history(namespace, threadId, { withTail? }) -> Promise` | `GET /namespaces/:namespace/workloads/:threadId/history?tail=1` | Read folded turns, optionally projecting the current Tier-1 tail as an incomplete turn snapshot. | +| Persistent Querying | `Agenetes.tail(namespace, threadId) -> AsyncIterable` | `GET /namespaces/:namespace/workloads/:threadId/events` | Follow the live Tier-1 event tail after the latest folded turn. | +| Configuration | `defineDriver(definition) -> MountedAgentDriver` | deployment / configuration API | Bind one driver's schema version, workload types, spec/state schemas, initial state, and typed implementation. | +| Configuration | `mountAgenetes({ drivers, ...stores }) -> Agenetes` | deployment / configuration API | Mount a complete static `kind → driver` map with instance-level persistence and recovery policy. | ## The Name: Agenetes / 名称：Agenetes @@ -72,6 +72,25 @@ The name is coined in the shape of its model, Kubernetes. Ancient Greek κυβε 这个名字是按 Kubernetes 的构词方式造出的。古希腊语 κυβερνήτης（_kubernḗtēs_，“舵手 / 治理者”）由词根 _kubern-_ 加施事后缀 **-ήτης (_-ētēs_)** 构成，意为“那个去做……的人”。Agenetes 中的 **ag- / agen-** 既保留了 “agent” 的可辨识性，也指向希腊语 ἄγω 与拉丁语 _agō_ → _agent_ 背后的“行动 / 驱动 / 引导”语义；结尾则对应同一个 **-ētēs** 施事后缀。因此它表达的不是简单的 `agen + netes` 切分，而是“驱动 agent / 使 agent workload 运转起来的人”——这正是 control plane 的工作。它的重音节奏也与其模型对应：Ku-ber-NÉ-tēs ⟷ A-ge-NÉ-tēs。 +## Asynchronous persistence + +ThreadStore, EventLogStore, and TurnStore implementations may return values or +promises. Agenetes awaits them. Callers must await `create`, `fork`, `rehome`, +`close`, `record`, `records`, `history`, and `logMetadata`; `get` remains a live +handle lookup, and `tail` and `notifications` remain async iterables. File and +in-memory stores retain their synchronous implementations. + +Writes complete before event publication and state notifications. Per-thread +state reports are queued, and `close` drains them. A state-persistence failure +is surfaced by subsequent record/run/close operations. Lifecycle operations +serialize across asynchronous rehome writes and compensation. Hosts must still +coordinate running turns and namespace changes; this is process-local ordering, +not a distributed transaction across stores. + +ThreadStore、EventLogStore、TurnStore 可以返回值或 Promise；Agenetes 会等待持久化完成。 +调用方须等待上述生命周期与持久化读取方法。事件与状态通知只在写入成功后发布， +`close` 会等待排队中的状态写入；这些保证仅限进程内，不构成跨存储的分布式事务。 + ## Core invariants (the design consensus) / 核心不变量（设计共识） The numbered invariants below (I1–I10, with sub-clauses I*n*._m_) are the design consensus, meant to be cited by reference id. Each is stated in English then Chinese; code blocks and tables are not duplicated. diff --git a/external/agenetes/packages/agenetes/src/event-log.test.ts b/external/agenetes/packages/agenetes/src/event-log.test.ts index 9df529f2e..44bdd23e1 100644 --- a/external/agenetes/packages/agenetes/src/event-log.test.ts +++ b/external/agenetes/packages/agenetes/src/event-log.test.ts @@ -224,43 +224,111 @@ describe('FileEventLogStore — on-disk specifics', () => { }); describe('EventLog — durable append + live pub/sub', () => { - it('persists via the store and fans out live entries to subscribers', () => { + it('persists via the store and fans out live entries to subscribers', async () => { const log = new EventLog(new InMemoryEventLogStore()); const n = ns('canvas-1'); // A pre-existing entry the subscriber must NOT receive (backfill is the // caller's read concern, not the live tail). - log.append(n, 'thread-1', text('before')); + await log.append(n, 'thread-1', text('before')); const seen: EventLogEntry[] = []; const unsub = log.subscribe('thread-1', (e) => seen.push(e)); - log.beginTurn(n, 'thread-1', request); - log.append(n, 'thread-1', text('live-1')); - log.append(n, 'thread-1', text('live-2')); + await log.beginTurn(n, 'thread-1', request); + await log.append(n, 'thread-1', text('live-1')); + await log.append(n, 'thread-1', text('live-2')); expect(seen.map((e) => e.seq)).toEqual([3, 4]); // Backfill is served by read, gap-free with the live tail. - expect(log.read(n, 'thread-1').map((e) => e.seq)).toEqual([1, 3, 4]); - expect(log.readRecords(n, 'thread-1').map((e) => e.seq)).toEqual([ + expect((await log.read(n, 'thread-1')).map((e) => e.seq)).toEqual([ + 1, 3, 4, + ]); + expect((await log.readRecords(n, 'thread-1')).map((e) => e.seq)).toEqual([ 1, 2, 3, 4, ]); unsub(); - log.append(n, 'thread-1', text('after-unsub')); + await log.append(n, 'thread-1', text('after-unsub')); expect(seen.map((e) => e.seq)).toEqual([3, 4]); }); - it('only notifies subscribers of the matching threadId', () => { + it('only notifies subscribers of the matching threadId', async () => { const log = new EventLog(new InMemoryEventLogStore()); const n = ns('canvas-1'); const seen: number[] = []; log.subscribe('thread-1', (e) => seen.push(e.seq)); - log.append(n, 'thread-2', text('other')); + await log.append(n, 'thread-2', text('other')); expect(seen).toEqual([]); - log.append(n, 'thread-1', text('mine')); + await log.append(n, 'thread-1', text('mine')); expect(seen).toEqual([1]); }); }); + +describe('asynchronous event persistence', () => { + it('orders concurrent appends and publishes only after the write commits', async () => { + const backing = new InMemoryEventLogStore(); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const store: EventLogStore = new Proxy(backing, { + get(target, key) { + if (key === 'append') + return async (...args: Parameters) => { + entered.resolve(); + await gate.promise; + return target.append(...args); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const log = new EventLog(store); + const namespace = ns('async'); + const seen: EventLogEntry[] = []; + log.subscribe('thread', (entry) => seen.push(entry)); + const first = log.append(namespace, 'thread', text('first')); + const second = log.append(namespace, 'thread', text('second')); + await entered.promise; + expect(seen).toEqual([]); + expect(backing.read(namespace, 'thread')).toEqual([]); + gate.resolve(); + expect( + (await Promise.all([first, second])).map((entry) => entry.seq), + ).toEqual([1, 2]); + expect(seen.map((entry) => entry.event)).toEqual([ + text('first'), + text('second'), + ]); + expect(await log.read(namespace, 'thread')).toEqual(seen); + }); + + it('surfaces rejected writes without publishing phantom events and allows retry', async () => { + const backing = new InMemoryEventLogStore(); + let fail = true; + const store: EventLogStore = new Proxy(backing, { + get(target, key) { + if (key === 'append') + return async (...args: Parameters) => { + await Promise.resolve(); + if (fail) throw new Error('durability unavailable'); + return target.append(...args); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const log = new EventLog(store); + const namespace = ns('async'); + const seen: EventLogEntry[] = []; + log.subscribe('thread', (entry) => seen.push(entry)); + await expect(log.append(namespace, 'thread', text('lost'))).rejects.toThrow( + 'durability unavailable', + ); + expect(seen).toEqual([]); + fail = false; + await log.append(namespace, 'thread', text('saved')); + expect(seen).toMatchObject([{ seq: 1, event: text('saved') }]); + }); +}); diff --git a/external/agenetes/packages/agenetes/src/event-log.ts b/external/agenetes/packages/agenetes/src/event-log.ts index b47a6a424..dcfdfcf85 100644 --- a/external/agenetes/packages/agenetes/src/event-log.ts +++ b/external/agenetes/packages/agenetes/src/event-log.ts @@ -85,7 +85,7 @@ export interface EventLogStore { namespace: Namespace, threadId: string, request: AgentSubmission | null, - ): TurnStartLogEntry; + ): TurnStartLogEntry | Promise; /** * Append `event` to the thread's log, assigning the next `seq` * (`maxSeq + 1`). Returns the durable entry, whose `seq` is the fence for @@ -95,7 +95,7 @@ export interface EventLogStore { namespace: Namespace, threadId: string, event: AgentStreamEvent, - ): EventLogEntry; + ): EventLogEntry | Promise; /** * Read the log for a thread. With `sinceSeq` set, returns only entries * with `seq > sinceSeq` (the fence read that powers `tail` reconnect); @@ -105,15 +105,15 @@ export interface EventLogStore { namespace: Namespace, threadId: string, sinceSeq?: number, - ): EventLogEntry[]; + ): EventLogEntry[] | Promise; /** Read both internal turn boundaries and streamed event entries. */ readRecords( namespace: Namespace, threadId: string, sinceSeq?: number, - ): EventLogRecord[]; + ): EventLogRecord[] | Promise; /** The highest `seq` persisted for a thread, or `0` when the log is empty. */ - maxSeq(namespace: Namespace, threadId: string): number; + maxSeq(namespace: Namespace, threadId: string): number | Promise; /** * Overwrite a thread's ENTIRE Tier-1 log with `records` (already-sequenced, * in original order), replacing whatever the target held before. A narrow @@ -125,13 +125,13 @@ export interface EventLogStore { namespace: Namespace, threadId: string, records: readonly EventLogRecord[], - ): void; + ): void | Promise; /** * Remove a thread's Tier-1 log entirely (idempotent). Reserved for the * `rehome()` primitive: dropping the source log after its target twin is * durable, or compensating a target log written during a failed rehome. */ - delete(namespace: Namespace, threadId: string): void; + delete(namespace: Namespace, threadId: string): void | Promise; } /** Defensive shape-check for a persisted entry read back from disk. */ @@ -389,6 +389,20 @@ export class EventLog { readonly #store: EventLogStore; readonly #listeners = new Map>(); + readonly #pending = new Map>(); + #serialize(threadId: string, operation: () => Promise): Promise { + const result = (this.#pending.get(threadId) ?? Promise.resolve()) + .catch(() => {}) + .then(operation); + this.#pending.set(threadId, result); + const clear = () => { + if (this.#pending.get(threadId) === result) + this.#pending.delete(threadId); + }; + void result.then(clear, clear); + return result; + } + constructor(store: EventLogStore) { this.#store = store; } @@ -397,50 +411,59 @@ export class EventLog { * Persist a turn boundary without publishing it to live event * subscribers. Public tail consumers observe only AgentStreamEvents. */ - beginTurn( + async beginTurn( namespace: Namespace, threadId: string, request: AgentSubmission | null, - ): TurnStartLogEntry { - return this.#store.appendTurnStart(namespace, threadId, request); + ): Promise { + return this.#serialize( + threadId, + async () => + await this.#store.appendTurnStart(namespace, threadId, request), + ); } /** Append + persist an event, then notify live subscribers of the thread. */ - append( + async append( namespace: Namespace, threadId: string, event: AgentStreamEvent, - ): EventLogEntry { - const entry = this.#store.append(namespace, threadId, event); - const set = this.#listeners.get(threadId); - if (set) { - // Snapshot so a listener that unsubscribes during dispatch is safe. - for (const listener of [...set]) listener(entry); - } - return entry; + ): Promise { + return this.#serialize(threadId, async () => { + const entry = await this.#store.append(namespace, threadId, event); + const set = this.#listeners.get(threadId); + if (set) { + // Snapshot so a listener that unsubscribes during dispatch is safe. + for (const listener of [...set]) listener(entry); + } + return entry; + }); } /** Durable read (fence read when `sinceSeq` is set); see {@link EventLogStore.read}. */ - read( + async read( namespace: Namespace, threadId: string, sinceSeq?: number, - ): EventLogEntry[] { - return this.#store.read(namespace, threadId, sinceSeq); + ): Promise { + await this.#pending.get(threadId); + return await this.#store.read(namespace, threadId, sinceSeq); } /** Durable read of all Tier-1 records for history materialization. */ - readRecords( + async readRecords( namespace: Namespace, threadId: string, sinceSeq?: number, - ): EventLogRecord[] { - return this.#store.readRecords(namespace, threadId, sinceSeq); + ): Promise { + await this.#pending.get(threadId); + return await this.#store.readRecords(namespace, threadId, sinceSeq); } /** The highest persisted `seq` for a thread (the fence), or `0` when empty. */ - maxSeq(namespace: Namespace, threadId: string): number { - return this.#store.maxSeq(namespace, threadId); + async maxSeq(namespace: Namespace, threadId: string): Promise { + await this.#pending.get(threadId); + return await this.#store.maxSeq(namespace, threadId); } /** @@ -449,20 +472,20 @@ export class EventLog { * write — it carries no live fan-out because a rehome target never has * subscribers yet (its thread does not exist before the move). */ - replace( + async replace( namespace: Namespace, threadId: string, records: readonly EventLogRecord[], - ): void { - this.#store.replace(namespace, threadId, records); + ): Promise { + await this.#store.replace(namespace, threadId, records); } /** * Remove a thread's Tier-1 log entirely; see {@link EventLogStore.delete}. * Reserved for `rehome()`'s source cleanup / target compensation. */ - delete(namespace: Namespace, threadId: string): void { - this.#store.delete(namespace, threadId); + async delete(namespace: Namespace, threadId: string): Promise { + await this.#store.delete(namespace, threadId); } /** diff --git a/external/agenetes/packages/agenetes/src/instance-log.test.ts b/external/agenetes/packages/agenetes/src/instance-log.test.ts index 840e925a5..0a001efc2 100644 --- a/external/agenetes/packages/agenetes/src/instance-log.test.ts +++ b/external/agenetes/packages/agenetes/src/instance-log.test.ts @@ -8,7 +8,7 @@ import { defineDriver } from '@agenetes/runtime'; import { describe, expect, it } from 'vitest'; -import { mountAgenetes } from './index.js'; +import { InMemoryTurnStore, type TurnStore, mountAgenetes } from './index.js'; import type { AgentCapabilities, @@ -109,7 +109,7 @@ async function drain(handle: AgentHandle, request: unknown): Promise { describe('Agenetes two-tier conversation log (M5.6/C3)', () => { it('folds a completed Deployment run into a Tier-2 AgentTurn (history)', async () => { const inst = mount(); - const handle = inst.create(deployment); + const handle = await inst.create(deployment); raw!.scripts.push({ events: [text('hi'), done('hi'), end()], result: [{ type: 'text', data: { content: 'hi' } }], @@ -117,14 +117,14 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { await drain(handle, { type: 'user_text', content: 'hello' }); - const { turns } = inst.history(ns, threadId); + const { turns } = await inst.history(ns, threadId); expect(turns).toHaveLength(1); expect(turns[0]!.request).toEqual({ type: 'user_text', content: 'hello' }); expect(turns[0]!.transcript).toEqual([ { type: 'text', data: { content: 'hi' } }, ]); expect(turns[0]!.meta).toEqual({ stopReason: 'end_turn' }); - expect(inst.logMetadata(ns, threadId)).toEqual({ + expect(await inst.logMetadata(ns, threadId)).toEqual({ eventCount: 4, turnCount: 1, }); @@ -132,7 +132,7 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { it('get(threadId) returns the same logging handle, so later turns fold too', async () => { const inst = mount(); - inst.create(deployment); + await inst.create(deployment); raw!.scripts.push({ events: [text('one'), end()], result: [{ type: 'text', data: { content: 'one' } }], @@ -145,7 +145,7 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { await drain(inst.get(threadId)!, { type: 'user_text', content: 'a' }); await drain(inst.get(threadId)!, { type: 'user_text', content: 'b' }); - const { turns } = inst.history(ns, threadId); + const { turns } = await inst.history(ns, threadId); expect(turns.map((t) => t.transcript[0]!.data)).toEqual([ { content: 'one' }, { content: 'two' }, @@ -154,7 +154,7 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { it('history({ withTail }) projects the in-flight turn, no seq leaked', async () => { const inst = mount(); - const handle = inst.create(deployment); + const handle = await inst.create(deployment); // Turn 1 completes → folded (fence at its last seq). raw!.scripts.push({ events: [text('committed'), end()], @@ -172,7 +172,9 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { { type: 'user_text', content: 'q2' } as never, {} as never, ); - expect(inst.history(ns, threadId, { withTail: true }).turns[1]).toEqual({ + expect( + (await inst.history(ns, threadId, { withTail: true })).turns[1], + ).toEqual({ request: { type: 'user_text', content: 'q2' }, transcript: [], isIncomplete: true, @@ -180,12 +182,12 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { await gen.next(); // yields live-a → Tier-1 append await gen.next(); // yields live-b → Tier-1 append - expect(inst.logMetadata(ns, threadId)).toEqual({ + expect(await inst.logMetadata(ns, threadId)).toEqual({ eventCount: 6, turnCount: 1, }); - const { turns } = inst.history(ns, threadId, { withTail: true }); + const { turns } = await inst.history(ns, threadId, { withTail: true }); expect(turns).toHaveLength(2); expect(turns[1]).toEqual({ request: { type: 'user_text', content: 'q2' }, @@ -203,7 +205,7 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { it('tail() ends when a terminal (end) frame is observed', async () => { const inst = mount(); - const handle = inst.create(deployment); + const handle = await inst.create(deployment); raw!.scripts.push({ events: [text('c'), end()], result: [{ type: 'text', data: { content: 'c' } }], @@ -231,7 +233,7 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { it('a live tail delivers events appended after it subscribes', async () => { const inst = mount(); - const handle = inst.create(deployment); + const handle = await inst.create(deployment); // Open the tail on a fresh thread (fence 0, empty backfill), then drive // a run so its frames arrive live. @@ -263,14 +265,14 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { namespace: ns, spec: {}, }; - const handle = inst.create(jobSpec); + const handle = await inst.create(jobSpec); raw!.scripts.push({ events: [text('job'), end()], result: [], }); await drain(handle, { type: 'user_text', content: 'go' }); - expect(inst.history(ns, 'thr_job').turns).toEqual([ + expect((await inst.history(ns, 'thr_job')).turns).toEqual([ { request: { type: 'user_text', content: 'go' }, transcript: [{ type: 'text', data: { content: 'job' } }], @@ -287,13 +289,50 @@ describe('Agenetes two-tier conversation log (M5.6/C3)', () => { namespace: ns, spec: {}, }; - const handle = inst.create(jobSpec); + const handle = await inst.create(jobSpec); raw!.scripts.push({ events: [text('transient'), end()], result: [], }); await drain(handle, { type: 'user_text', content: 'go' }); - expect(inst.history(ns, '').turns).toEqual([]); + expect((await inst.history(ns, '')).turns).toEqual([]); }); }); + +it('keeps live frames when a remote fence read completes after the turn is folded', async () => { + const turns = new InMemoryTurnStore(); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const turnStore: TurnStore = new Proxy(turns, { + get(target, key) { + if (key === 'fence') + return async (...args: Parameters) => { + entered.resolve(); + await gate.promise; + return target.fence(...args); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const inst = mountAgenetes({ + drivers: { external: scriptedDriver() }, + turnStore, + }); + const handle = await inst.create(deployment); + const tail = inst.tail(ns, threadId)[Symbol.asyncIterator](); + const first = tail.next(); + await entered.promise; + raw!.scripts.push({ + events: [text('while reading fence'), end()], + result: [], + }); + await drain(handle, { type: 'user_text', content: 'question' }); + expect(turns.fence(ns, threadId)).toBeGreaterThan(0); + gate.resolve(); + expect((await first).value).toEqual(text('while reading fence')); + expect((await tail.next()).value).toEqual(end()); + expect((await tail.next()).done).toBe(true); + await inst.close(threadId); +}); diff --git a/external/agenetes/packages/agenetes/src/instance.rehome.test.ts b/external/agenetes/packages/agenetes/src/instance.rehome.test.ts index 7c6180208..0da5324fb 100644 --- a/external/agenetes/packages/agenetes/src/instance.rehome.test.ts +++ b/external/agenetes/packages/agenetes/src/instance.rehome.test.ts @@ -142,7 +142,7 @@ const targetSpecFor = ( }); describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { - it('relocates the thread record, Tier-1 events, and Tier-2 turns to the target namespace', () => { + it('relocates the thread record, Tier-1 events, and Tier-2 turns to the target namespace', async () => { const threadStore = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const turnStore = new InMemoryTurnStore(); @@ -171,11 +171,11 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { sourceRecord.spec as StubSpec, targetNamespace, ); - inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec); + await inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec); // Target: the visible durable owner with the rewritten namespace/spec, // preserved threadId, driver kind, workload type, and driver state. - expect(inst.record(targetNamespace, threadId)).toEqual({ + expect(await inst.record(targetNamespace, threadId)).toEqual({ driverSchemaVersion: 1, spec: targetSpec, state: sourceRecord.state, @@ -184,12 +184,12 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { sourceEvents, ); expect(turnStore.list(targetNamespace, threadId)).toEqual(sourceTurns); - expect(inst.history(targetNamespace, threadId).turns).toEqual( + expect((await inst.history(targetNamespace, threadId)).turns).toEqual( sourceTurns.map((p) => p.turn), ); // Source: fully removed — record, Tier-1 log, and Tier-2 log. - expect(inst.record(sourceNamespace, threadId)).toBeUndefined(); + expect(await inst.record(sourceNamespace, threadId)).toBeUndefined(); expect(eventLogStore.readRecords(sourceNamespace, threadId)).toEqual([]); expect(turnStore.list(sourceNamespace, threadId)).toEqual([]); @@ -197,7 +197,7 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { expect(inst.get(threadId)).toBeUndefined(); }); - it('rejects a source thread with a live handle, leaving source and target untouched', () => { + it('rejects a source thread with a live handle, leaving source and target untouched', async () => { const threadStore = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const turnStore = new InMemoryTurnStore(); @@ -218,18 +218,19 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { spec: {}, }; // create() spawns a live Deployment handle and upserts the record. - inst.create(sourceSpec); + await inst.create(sourceSpec); const targetSpec = targetSpecFor(sourceSpec, targetNamespace); - expect(() => - inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec), - ).toThrow(/live handle/); - expect(inst.record(targetNamespace, threadId)).toBeUndefined(); - expect(inst.record(sourceNamespace, threadId)).toBeDefined(); + await expect( + async () => + await inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec), + ).rejects.toThrow(/live handle/); + expect(await inst.record(targetNamespace, threadId)).toBeUndefined(); + expect(await inst.record(sourceNamespace, threadId)).toBeDefined(); expect(inst.get(threadId)).toBeDefined(); }); - it('rejects a missing source thread', () => { + it('rejects a missing source thread', async () => { const inst = mountAgenetes({ drivers: { external: stubDriver() } }); const namespace = ns('canvas_1'); const targetSpec: StubSpec = { @@ -239,9 +240,10 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { namespace: ns('canvas_2'), spec: {}, }; - expect(() => - inst.rehome({ namespace, threadId: 'missing' }, targetSpec), - ).toThrow(/missing source thread/); + await expect( + async () => + await inst.rehome({ namespace, threadId: 'missing' }, targetSpec), + ).rejects.toThrow(/missing source thread/); }); it.each([ @@ -250,7 +252,7 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { ['conflicting Tier-1 events', 'events'], ] as const)( 'rejects a target namespace with %s, leaving source untouched', - (_label, conflictKind) => { + async (_label, conflictKind) => { const threadStore = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const turnStore = new InMemoryTurnStore(); @@ -294,11 +296,17 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { sourceRecord.spec as StubSpec, targetNamespace, ); - expect(() => - inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec), - ).toThrow(/already exists/); + await expect( + async () => + await inst.rehome( + { namespace: sourceNamespace, threadId }, + targetSpec, + ), + ).rejects.toThrow(/already exists/); // The source is completely untouched by a rejected precondition. - expect(inst.record(sourceNamespace, threadId)).toEqual(sourceRecord); + expect(await inst.record(sourceNamespace, threadId)).toEqual( + sourceRecord, + ); expect(eventLogStore.readRecords(sourceNamespace, threadId).length).toBe( 3, ); @@ -306,7 +314,7 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { }, ); - it('rejects a target threadId, driver kind, or workload type that differs from the source', () => { + it('rejects a target threadId, driver kind, or workload type that differs from the source', async () => { const threadStore = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const turnStore = new InMemoryTurnStore(); @@ -329,33 +337,37 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { }); const base = targetSpecFor(sourceRecord.spec as StubSpec, targetNamespace); - expect(() => - inst.rehome( - { namespace: sourceNamespace, threadId }, - { ...base, threadId: 'renamed' }, - ), - ).toThrow(/threadId must equal source/); - expect(() => - inst.rehome( - { namespace: sourceNamespace, threadId }, - { ...base, kind: 'internal' }, - ), - ).toThrow(/driver kind must match source/); - expect(() => - inst.rehome( - { namespace: sourceNamespace, threadId }, - { ...base, workloadType: 'Job' }, - ), - ).toThrow(/workload type must match source/); - expect(() => - inst.rehome( - { namespace: sourceNamespace, threadId }, - { ...base, namespace: sourceNamespace }, - ), - ).toThrow(/namespace must differ from source/); + await expect( + async () => + await inst.rehome( + { namespace: sourceNamespace, threadId }, + { ...base, threadId: 'renamed' }, + ), + ).rejects.toThrow(/threadId must equal source/); + await expect( + async () => + await inst.rehome( + { namespace: sourceNamespace, threadId }, + { ...base, kind: 'internal' }, + ), + ).rejects.toThrow(/driver kind must match source/); + await expect( + async () => + await inst.rehome( + { namespace: sourceNamespace, threadId }, + { ...base, workloadType: 'Job' }, + ), + ).rejects.toThrow(/workload type must match source/); + await expect( + async () => + await inst.rehome( + { namespace: sourceNamespace, threadId }, + { ...base, namespace: sourceNamespace }, + ), + ).rejects.toThrow(/namespace must differ from source/); }); - it('restores the source and removes every target write on a determinate failure', () => { + it('restores the source and removes every target write on a determinate failure', async () => { const threadStore = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const turnStore = new InMemoryTurnStore(); @@ -402,9 +414,10 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { targetNamespace, ); - expect(() => - inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec), - ).toThrow(/simulated target record write failure/); + await expect( + async () => + await inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec), + ).rejects.toThrow(/simulated target record write failure/); expect(upsertCalls).toBe(1); // Target logs written during the attempt are fully rolled back. @@ -420,7 +433,7 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { expect(turnStore.list(sourceNamespace, threadId)).toEqual(sourceTurns); }); - it('reports a distinct unknown-outcome error when the rollback itself fails', () => { + it('reports a distinct unknown-outcome error when the rollback itself fails', async () => { const threadStore = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const realTurnStore = new InMemoryTurnStore(); @@ -470,7 +483,7 @@ describe('Agenetes.rehome() — the destructive counterpart to fork()', () => { let caught: unknown; try { - inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec); + await inst.rehome({ namespace: sourceNamespace, threadId }, targetSpec); } catch (error) { caught = error; } diff --git a/external/agenetes/packages/agenetes/src/instance.test.ts b/external/agenetes/packages/agenetes/src/instance.test.ts index ddca2316d..363722b4d 100644 --- a/external/agenetes/packages/agenetes/src/instance.test.ts +++ b/external/agenetes/packages/agenetes/src/instance.test.ts @@ -84,7 +84,7 @@ function mount() { } describe('mounted Agenetes instance (M5 INST skeleton)', () => { - it('create() get-or-creates by threadId and reuse ignores spec (I9.3)', () => { + it('create() get-or-creates by threadId and reuse ignores spec (I9.3)', async () => { const inst = mount(); const spec: StubSpec = { threadId: 'thr_1', @@ -93,17 +93,17 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace: ns('canvas_1', '/data/c1'), spec: { note: 'first' }, }; - const h1 = inst.create(spec) as unknown as StubHandle; - const h2 = inst.create({ + const h1 = (await inst.create(spec)) as unknown as StubHandle; + const h2 = (await inst.create({ ...spec, spec: { note: 'second' }, - }) as unknown as StubHandle; + })) as unknown as StubHandle; expect(h2).toBe(h1); // reuse-ignores-spec: the live handle keeps its original spec expect(h1.spec.spec.note).toBe('first'); }); - it('restart recovery keeps the persisted spec authoritative', () => { + it('restart recovery keeps the persisted spec authoritative', async () => { const inst = mount(); const spec: StubSpec = { threadId: 'thr_1', @@ -112,22 +112,24 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace: ns('canvas_1', '/data/c1'), spec: { note: 'persisted' }, }; - inst.create(spec); - inst.close(spec.threadId); + await inst.create(spec); + await inst.close(spec.threadId); - const recovered = inst.create({ + const recovered = (await inst.create({ ...spec, spec: { note: 'drifted' }, - }) as unknown as StubHandle; + })) as unknown as StubHandle; expect(recovered.spec.spec.note).toBe('persisted'); - expect(inst.record(spec.namespace, spec.threadId)?.spec.spec).toEqual( + expect( + (await inst.record(spec.namespace, spec.threadId))?.spec.spec, + ).toEqual( expect.objectContaining({ note: 'persisted', }), ); }); - it('rejects changing the driver kind of a persisted thread', () => { + it('rejects changing the driver kind of a persisted thread', async () => { const store = new InMemoryThreadStore(); const first = mountAgenetes({ drivers: { external: stubDriver(), internal: stubDriver() }, @@ -140,19 +142,19 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace: ns('canvas_1'), spec: {}, }; - first.create(spec); - first.close(spec.threadId); + await first.create(spec); + await first.close(spec.threadId); const restarted = mountAgenetes({ drivers: { external: stubDriver(), internal: stubDriver() }, threadStore: store, }); - expect(() => restarted.create({ ...spec, kind: 'internal' })).toThrow( - /cannot change driver kind/, - ); + await expect( + async () => await restarted.create({ ...spec, kind: 'internal' }), + ).rejects.toThrow(/cannot change driver kind/); }); - it('fork() realizes a fresh target from source durable input', () => { + it('fork() realizes a fresh target from source durable input', async () => { const store = new InMemoryThreadStore(); const eventLogStore = new InMemoryEventLogStore(); const turnStore = new InMemoryTurnStore(); @@ -212,10 +214,10 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { spec: { note: 'complete target' }, }; - const handle = inst.fork( + const handle = (await inst.fork( { namespace: sourceNamespace, threadId: sourceSpec.threadId }, targetSpec, - ) as unknown as StubHandle; + )) as unknown as StubHandle; expect(handle.spec).toEqual(targetSpec); expect(handle.createContext.recoveryInput).toBeUndefined(); @@ -233,20 +235,20 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { }, ], }); - expect(inst.record(targetNamespace, targetSpec.threadId)).toEqual({ + expect(await inst.record(targetNamespace, targetSpec.threadId)).toEqual({ driverSchemaVersion: 1, spec: targetSpec, state: { driverState: {} }, }); - expect(inst.history(targetNamespace, targetSpec.threadId).turns).toEqual( - [], - ); - expect(inst.record(sourceNamespace, sourceSpec.threadId)?.state).toEqual( - sourceState, - ); + expect( + (await inst.history(targetNamespace, targetSpec.threadId)).turns, + ).toEqual([]); + expect( + (await inst.record(sourceNamespace, sourceSpec.threadId))?.state, + ).toEqual(sourceState); }); - it('fork() rejects a missing source and non-fresh target', () => { + it('fork() rejects a missing source and non-fresh target', async () => { const inst = mount(); const namespace = ns('canvas_1'); const targetSpec: StubSpec = { @@ -256,27 +258,30 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace, spec: {}, }; - expect(() => - inst.fork({ namespace, threadId: 'missing' }, targetSpec), - ).toThrow(/missing source thread/); + await expect( + async () => + await inst.fork({ namespace, threadId: 'missing' }, targetSpec), + ).rejects.toThrow(/missing source thread/); - inst.create({ + await inst.create({ ...targetSpec, threadId: 'source_thread', }); - expect(() => - inst.fork( - { namespace, threadId: 'source_thread' }, - { ...targetSpec, threadId: 'source_thread' }, - ), - ).toThrow(/target threadId must differ/); - inst.create(targetSpec); - expect(() => - inst.fork({ namespace, threadId: 'source_thread' }, targetSpec), - ).toThrow(/target thread already exists/); + await expect( + async () => + await inst.fork( + { namespace, threadId: 'source_thread' }, + { ...targetSpec, threadId: 'source_thread' }, + ), + ).rejects.toThrow(/target threadId must differ/); + await inst.create(targetSpec); + await expect( + async () => + await inst.fork({ namespace, threadId: 'source_thread' }, targetSpec), + ).rejects.toThrow(/target thread already exists/); }); - it('get() is a pure lookup that never spawns (I9.3)', () => { + it('get() is a pure lookup that never spawns (I9.3)', async () => { const inst = mount(); expect(inst.get('missing')).toBeUndefined(); const spec: StubSpec = { @@ -286,25 +291,25 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace: ns('canvas_1'), spec: {}, }; - const created = inst.create(spec); + const created = await inst.create(spec); expect(inst.get('thr_1')).toBe(created); }); - it('close() tears the handle down and evicts it (I9.3)', () => { + it('close() tears the handle down and evicts it (I9.3)', async () => { const inst = mount(); - const handle = inst.create({ + const handle = (await inst.create({ threadId: 'thr_1', kind: 'external', workloadType: 'Deployment', namespace: ns('canvas_1'), spec: {}, - }) as unknown as StubHandle; - inst.close('thr_1'); + })) as unknown as StubHandle; + await inst.close('thr_1'); expect(handle.closed).toBe(true); expect(inst.get('thr_1')).toBeUndefined(); }); - it('a Job is minted fresh each turn and never enters the live table (I3.2/I9.3)', () => { + it('a Job is minted fresh each turn and never enters the live table (I3.2/I9.3)', async () => { const inst = mount(); const spec: StubSpec = { threadId: 'thr_job', @@ -313,11 +318,11 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace: ns('canvas_1', '/data/c1'), spec: { note: 'first' }, }; - const h1 = inst.create(spec) as unknown as StubHandle; - const h2 = inst.create({ + const h1 = (await inst.create(spec)) as unknown as StubHandle; + const h2 = (await inst.create({ ...spec, spec: { note: 'second' }, - }) as unknown as StubHandle; + })) as unknown as StubHandle; // distinct handles — a Job is not cached / reused expect(h2).not.toBe(h1); expect(h1.spec.spec.note).toBe('first'); @@ -325,12 +330,12 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { // and it never registers in the live-handle table expect(inst.get('thr_job')).toBeUndefined(); // but the durable record is still upserted (query surface, I9.4) - expect(inst.record(spec.namespace, 'thr_job')?.spec.spec).toEqual( + expect((await inst.record(spec.namespace, 'thr_job'))?.spec.spec).toEqual( expect.objectContaining({ note: 'second' }), ); }); - it('a transient Job (empty threadId) upserts no durable record (I9.4)', () => { + it('a transient Job (empty threadId) upserts no durable record (I9.4)', async () => { const inst = mount(); const namespace = ns('canvas_1', '/data/c1'); const spec: StubSpec = { @@ -341,28 +346,29 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { spec: { note: 'stateless' }, }; // it still runs and returns a fresh handle … - const handle = inst.create(spec) as unknown as StubHandle; + const handle = (await inst.create(spec)) as unknown as StubHandle; expect(handle.spec.spec.note).toBe('stateless'); // … but leaves no durable footprint: an empty key would collide across // every transient Job in the namespace and accumulate junk records. - expect(inst.record(namespace, '')).toBeUndefined(); - expect(inst.records(namespace)).toEqual([]); + expect(await inst.record(namespace, '')).toBeUndefined(); + expect(await inst.records(namespace)).toEqual([]); }); - it('create() dispatches on spec.kind; unknown kind throws', () => { + it('create() dispatches on spec.kind; unknown kind throws', async () => { const inst = mount(); - expect(() => - inst.create({ - threadId: 'thr_x', - kind: 'nope', - workloadType: 'Deployment', - namespace: ns('canvas_1'), - spec: {}, - }), - ).toThrow(/no agent driver mounted for kind 'nope'/); + await expect( + async () => + await inst.create({ + threadId: 'thr_x', + kind: 'nope', + workloadType: 'Deployment', + namespace: ns('canvas_1'), + spec: {}, + }), + ).rejects.toThrow(/no agent driver mounted for kind 'nope'/); }); - it('query surface reads durable records, orthogonal to liveness (I9.4)', () => { + it('query surface reads durable records, orthogonal to liveness (I9.4)', async () => { const inst = mount(); const namespace = ns('canvas_1', '/data/c1'); const spec: StubSpec = { @@ -372,31 +378,31 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { namespace, spec: {}, }; - inst.create(spec); + await inst.create(spec); - const rec = inst.record(namespace, 'thr_1'); + const rec = await inst.record(namespace, 'thr_1'); expect(rec?.spec).toEqual(spec); expect(rec?.driverSchemaVersion).toBe(1); expect(rec?.state).toEqual({ driverState: {} }); // closing the live handle does NOT drop the durable record - inst.close('thr_1'); + await inst.close('thr_1'); expect(inst.get('thr_1')).toBeUndefined(); - expect(inst.record(namespace, 'thr_1')?.spec).toEqual(spec); + expect((await inst.record(namespace, 'thr_1'))?.spec).toEqual(spec); }); - it('durable records are isolated per namespace (I4.1 / I9.4)', () => { + it('durable records are isolated per namespace (I4.1 / I9.4)', async () => { const inst = mount(); const nsA = ns('canvas_A', '/data/a'); const nsB = ns('canvas_B', '/data/b'); - inst.create({ + await inst.create({ threadId: 'thr_a', kind: 'external', workloadType: 'Deployment', namespace: nsA, spec: {}, }); - inst.create({ + await inst.create({ threadId: 'thr_b', kind: 'external', workloadType: 'Deployment', @@ -404,12 +410,16 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { spec: {}, }); - expect(inst.records(nsA).map((r) => r.spec.threadId)).toEqual(['thr_a']); - expect(inst.records(nsB).map((r) => r.spec.threadId)).toEqual(['thr_b']); - expect(inst.record(nsA, 'thr_b')).toBeUndefined(); + expect((await inst.records(nsA)).map((r) => r.spec.threadId)).toEqual([ + 'thr_a', + ]); + expect((await inst.records(nsB)).map((r) => r.spec.threadId)).toEqual([ + 'thr_b', + ]); + expect(await inst.record(nsA, 'thr_b')).toBeUndefined(); }); - it('down-feeds the durable snapshot into driver.create and preserves it on reuse (I9.7)', () => { + it('down-feeds the durable snapshot into driver.create and preserves it on reuse (I9.7)', async () => { const store = new InMemoryThreadStore(); const turnStore = new InMemoryTurnStore(); const namespace = ns('canvas_1', '/data/c1'); @@ -452,33 +462,34 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { spec: {}, }; // Down-feed: the driver receives the durable record at create time. - const handle = inst.create(spec) as unknown as StubHandle; + const handle = (await inst.create(spec)) as unknown as StubHandle; expect(handle.createContext.forkInput).toBeUndefined(); expect(handle.createContext.recoveryInput?.state).toEqual(prior); expect(handle.createContext.recoveryInput?.turns).toEqual([foldedTurn]); // The state-preserving upsert must NOT clobber the persisted snapshot // back to `{}` — a returning thread keeps its resume token + metadata. - expect(inst.record(namespace, 'thr_1')?.state).toEqual(prior); + expect((await inst.record(namespace, 'thr_1'))?.state).toEqual(prior); // Reuse (get-or-create) also leaves the durable state intact. - inst.create(spec); - expect(inst.record(namespace, 'thr_1')?.state).toEqual(prior); + await inst.create(spec); + expect((await inst.record(namespace, 'thr_1'))?.state).toEqual(prior); }); - it('create() throws when no driver is mounted for the requested kind', () => { - expect(() => - mountAgenetes({ drivers: {} }).create({ - threadId: 'thr_1', - kind: 'external', - workloadType: 'Deployment', - namespace: ns('canvas_1'), - spec: {}, - }), - ).toThrow(/no agent driver mounted for kind 'external'/); + it('create() throws when no driver is mounted for the requested kind', async () => { + await expect( + async () => + await mountAgenetes({ drivers: {} }).create({ + threadId: 'thr_1', + kind: 'external', + workloadType: 'Deployment', + namespace: ns('canvas_1'), + spec: {}, + }), + ).rejects.toThrow(/no agent driver mounted for kind 'external'/); }); - it('injected ThreadStore backs the query surface (I9.4 port)', () => { + it('injected ThreadStore backs the query surface (I9.4 port)', async () => { const upsert = vi.fn(); const list = vi.fn().mockReturnValue([]); const inst = mountAgenetes({ @@ -492,7 +503,7 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { }); const namespace = ns('canvas_1'); - inst.create({ + await inst.create({ threadId: 'thr_1', kind: 'external', workloadType: 'Deployment', @@ -500,7 +511,7 @@ describe('mounted Agenetes instance (M5 INST skeleton)', () => { spec: {}, }); expect(upsert).toHaveBeenCalledTimes(1); - inst.records(namespace); + await inst.records(namespace); expect(list).toHaveBeenCalledWith(namespace); }); }); diff --git a/external/agenetes/packages/agenetes/src/instance.ts b/external/agenetes/packages/agenetes/src/instance.ts index 7bf80e4c0..de5ca778e 100644 --- a/external/agenetes/packages/agenetes/src/instance.ts +++ b/external/agenetes/packages/agenetes/src/instance.ts @@ -74,14 +74,14 @@ export interface Agenetes { * Either way the durable thread record is upserted so the query surface * can read it independent of handle liveness (I9.4). */ - create(spec: WorkloadSpec): AgentHandle; + create(spec: WorkloadSpec): Promise; /** * Realise a fresh target thread from a durable source thread. The host * supplies the complete target spec; Agenetes performs no field-level * merge. The target receives the source record and folded turns but * starts with an empty target state. */ - fork(source: ThreadIdentity, targetSpec: WorkloadSpec): AgentHandle; + fork(source: ThreadIdentity, targetSpec: WorkloadSpec): Promise; /** * The destructive counterpart to {@link Agenetes.fork}: relocate a * durable thread's complete conversation ownership — its thread record, @@ -115,7 +115,7 @@ export interface Agenetes { * "unknown, do not assume the source is intact" rather than a normal * determinate failure. */ - rehome(source: ThreadIdentity, targetSpec: WorkloadSpec): void; + rehome(source: ThreadIdentity, targetSpec: WorkloadSpec): Promise; /** * Pure lookup of the live handle for `threadId` — **never spawns** * (I9.3). A missing handle is a precondition failure (e.g. a control @@ -123,14 +123,17 @@ export interface Agenetes { */ get(threadId: string): AgentHandle | undefined; /** Tear the live handle down and evict it from the live table (I9.3). */ - close(threadId: string): void; + close(threadId: string): Promise; /** * Read one durable thread record by `(namespace, threadId)` (I9.4), * independent of whether a handle is live. */ - record(namespace: Namespace, threadId: string): ThreadRecord | undefined; + record( + namespace: Namespace, + threadId: string, + ): Promise; /** Enumerate a namespace's persisted thread records (I9.4). */ - records(namespace: Namespace): ThreadRecord[]; + records(namespace: Namespace): Promise; /** * The notification surface (I9.7): subscribe to a thread's driver-agnostic * `AgentMetadata` as it changes. The instance persists each up-reported @@ -144,7 +147,10 @@ export interface Agenetes { * Read lightweight metadata about the two-tier conversation log without * loading its events or folded turns. */ - logMetadata(namespace: Namespace, threadId: string): ThreadLogMetadata; + logMetadata( + namespace: Namespace, + threadId: string, + ): Promise; /** * Read a thread's durable conversation as folded {@link AgentTurn}s * (Tier 2 of the two-tier log, README I9.8). With `withTail`, the @@ -155,7 +161,7 @@ export interface Agenetes { namespace: Namespace, threadId: string, options?: HistoryOptions, - ): ThreadHistory; + ): Promise; /** * Follow a thread's LIVE tail: the Tier-1 events appended after the last * folded turn (the uncommitted in-flight turn, replayed on connect) plus @@ -219,7 +225,7 @@ function createTail( eventLog: EventLog, namespace: Namespace, threadId: string, - sinceSeq: number, + readFence: () => number | Promise, ): AsyncIterable { return { [Symbol.asyncIterator](): AsyncIterator { @@ -228,7 +234,7 @@ function createTail( null; let finished = false; // The highest seq already delivered; the dedup / resume watermark. - let lastSeq = sinceSeq; + let lastSeq = 0; let unsub: () => void = () => {}; const finish = (): void => { @@ -253,7 +259,17 @@ function createTail( }); // Snapshot the already-persisted tail (entries after the fence). - const backfill = eventLog.read(namespace, threadId, sinceSeq); + let backfill: EventLogEntry[] = []; + const readyBackfill = (async () => { + const fence = await readFence(); + const persisted = await eventLog.read(namespace, threadId, fence); + // A turn can finish while a remote fence read is in flight. Keep + // events observed live even if that fence now includes their turn. + backfill = [...persisted, ...live.splice(0)].sort( + (a, b) => a.seq - b.seq, + ); + })(); + void readyBackfill.catch(() => finish()); let backfillIdx = 0; // Pull the next not-yet-delivered entry: backfill first (ascending @@ -279,12 +295,11 @@ function createTail( }; return { - next(): Promise> { + async next(): Promise> { + await readyBackfill; + if (finished) return { value: undefined, done: true }; const ready = pump(); - if (ready) return Promise.resolve(ready); - if (finished) { - return Promise.resolve({ value: undefined, done: true }); - } + if (ready) return ready; return new Promise((resolve) => { waiting = resolve; }); @@ -389,17 +404,17 @@ export function createAgenetesInstance( }; }; - const readHistory = ( + const readHistory = async ( namespace: Namespace, threadId: string, withTail: boolean, - ): ObservedAgentTurn[] => { - const persisted = turnStore.list(namespace, threadId); + ): Promise => { + const persisted = await turnStore.list(namespace, threadId); if (!withTail) return persisted.map(({ turn }) => turn); const fence = persisted[persisted.length - 1]?.seqEnd ?? 0; return materializeHistory( persisted, - eventLog.readRecords(namespace, threadId, fence), + await eventLog.readRecords(namespace, threadId, fence), ); }; @@ -407,20 +422,29 @@ export function createAgenetesInstance( // FIRST (sole writer), then re-emit its metadata (persist-then-notify). // A handle without `onState` (a driver that reports no out-of-turn meta) // wires nothing and its notification stream stays empty. + const pendingReports = new Map>(); + const wireUpReport = ( spec: WorkloadSpec, driver: MountedAgentDriver, handle: AgentHandle, ): void => { const unsub = handle.onState?.((snapshot: AgentStateSnapshot) => { - threadStore.upsert(spec.namespace, spec.threadId, { - driverSchemaVersion: driver.schemaVersion, - spec, - state: snapshot, + const pending = ( + pendingReports.get(spec.threadId) ?? Promise.resolve() + ).then(async () => { + await threadStore.upsert(spec.namespace, spec.threadId, { + driverSchemaVersion: driver.schemaVersion, + spec, + state: snapshot, + }); + if (snapshot.metadata !== undefined) + bus.publish(spec.threadId, snapshot.metadata); }); - if (snapshot.metadata !== undefined) { - bus.publish(spec.threadId, snapshot.metadata); - } + pendingReports.set(spec.threadId, pending); + // Driver callbacks cannot await. Retain failures for record/run/close to + // surface; attach a handler immediately so no rejection is unobserved. + void pending.catch(() => {}); }); if (unsub) unsubscribers.set(spec.threadId, unsub); }; @@ -456,26 +480,34 @@ export function createAgenetesInstance( // the deltas the driver already yields, so `TResult` stays free. const folder = createTranscriptFolder(); let meta: AgentTurnMeta | undefined; - let step = await source.next(); - while (!step.done) { - const event = step.value; - eventLog.append(namespace, threadId, event); - folder.fold(event); - if (event.type === AGENT_STREAM_EVENTS.Done) meta = event.data.meta; - yield event; - step = await source.next(); + let completed = false; + try { + let step = await source.next(); + while (!step.done) { + await pendingReports.get(threadId); + const event = step.value; + await eventLog.append(namespace, threadId, event); + folder.fold(event); + if (event.type === AGENT_STREAM_EVENTS.Done) meta = event.data.meta; + yield event; + step = await source.next(); + } + // The generator returned. Commit the Tier-2 turn pinned to its Tier-1 + // range, then pass the raw return value through UNCHANGED so the host's + // own consumer still sees whatever `TResult` the driver produced. + await pendingReports.get(threadId); + const seqEnd = await eventLog.maxSeq(namespace, threadId); + const turn: AgentTurn = { + request: start.request, + transcript: folder.result(), + ...(meta ? { meta } : {}), + }; + await turnStore.append(namespace, threadId, { turn, seqStart, seqEnd }); + completed = true; + return step.value; + } finally { + if (!completed) await source.return(undefined); } - // The generator returned. Commit the Tier-2 turn pinned to its Tier-1 - // range, then pass the raw return value through UNCHANGED so the host's - // own consumer still sees whatever `TResult` the driver produced. - const seqEnd = eventLog.maxSeq(namespace, threadId); - const turn: AgentTurn = { - request: start.request, - transcript: folder.result(), - ...(meta ? { meta } : {}), - }; - turnStore.append(namespace, threadId, { turn, seqStart, seqEnd }); - return step.value; } return new Proxy(inner, { @@ -485,20 +517,25 @@ export function createAgenetesInstance( submission: unknown, ctx: unknown, ): AsyncGenerator => { + // Starting run() records the boundary immediately, even before the + // generator is pulled; recovery can observe an empty in-flight turn. const start = eventLog.beginTurn( namespace, threadId, coerceSubmission(submission), ); - return loggingRun( - ( + void start.catch(() => {}); + return (async function* () { + await pendingReports.get(threadId); + const boundary = await start; + const source = ( target.run as ( s: unknown, c: unknown, ) => AsyncGenerator - )(submission, ctx), - start, - ); + )(submission, ctx); + return yield* loggingRun(source, boundary); + })(); }; } // Forward every other member to the backing handle. Bind methods to @@ -510,12 +547,21 @@ export function createAgenetesInstance( }); }; - const realize = ( + const realize = async ( targetSpec: WorkloadSpec, driver: MountedAgentDriver, context: AgentCreateContext, initialState: AgentStateSnapshot, - ): AgentHandle => { + ): Promise => { + const isTransientJob = + targetSpec.workloadType === 'Job' && !targetSpec.threadId; + if (!isTransientJob) { + await threadStore.upsert(targetSpec.namespace, targetSpec.threadId, { + driverSchemaVersion: driver.schemaVersion, + spec: targetSpec, + state: initialState, + }); + } let handle: AgentHandle; if (targetSpec.workloadType === 'Job') { const raw = driver.create(targetSpec, context); @@ -535,25 +581,16 @@ export function createAgenetesInstance( if (!wasLive) wireUpReport(targetSpec, driver, handle); } - const isTransientJob = - targetSpec.workloadType === 'Job' && !targetSpec.threadId; - if (!isTransientJob) { - threadStore.upsert(targetSpec.namespace, targetSpec.threadId, { - driverSchemaVersion: driver.schemaVersion, - spec: targetSpec, - state: initialState, - }); - } return handle; }; - return { - create(rawSpec: WorkloadSpec): AgentHandle { + const api: Agenetes = { + async create(rawSpec: WorkloadSpec): Promise { const incoming = validateSpec(rawSpec); // A persisted same-thread spec is authoritative across restart, // preserving reuse-ignores-spec semantics when no live handle exists. const rawPrior = incoming.spec.threadId - ? threadStore.get(incoming.spec.namespace, incoming.spec.threadId) + ? await threadStore.get(incoming.spec.namespace, incoming.spec.threadId) : undefined; const prior = rawPrior ? validateRecord(rawPrior) : undefined; if (prior && prior.spec.kind !== incoming.spec.kind) { @@ -575,7 +612,7 @@ export function createAgenetesInstance( recovery, recoveryInput: { state: prior.state, - turns: readHistory( + turns: await readHistory( incoming.spec.namespace, incoming.spec.threadId, true, @@ -588,10 +625,16 @@ export function createAgenetesInstance( ({ driverState: target.driver.initialState(), } satisfies AgentStateSnapshot); - return realize(target.spec, target.driver, context, initialState); + return await realize(target.spec, target.driver, context, initialState); }, - fork(source: ThreadIdentity, rawTargetSpec: WorkloadSpec): AgentHandle { - const sourceRecord = threadStore.get(source.namespace, source.threadId); + async fork( + source: ThreadIdentity, + rawTargetSpec: WorkloadSpec, + ): Promise { + const sourceRecord = await threadStore.get( + source.namespace, + source.threadId, + ); if (!sourceRecord) { throw new AgenetesError( 'invalid_workload', @@ -609,9 +652,10 @@ export function createAgenetesInstance( } if ( runtime.get(targetSpec.threadId) !== undefined || - threadStore.get(targetSpec.namespace, targetSpec.threadId) !== + (await threadStore.get(targetSpec.namespace, targetSpec.threadId)) !== undefined || - turnStore.list(targetSpec.namespace, targetSpec.threadId).length > 0 + (await turnStore.list(targetSpec.namespace, targetSpec.threadId)) + .length > 0 ) { throw new AgenetesError( 'invalid_workload', @@ -624,27 +668,33 @@ export function createAgenetesInstance( 'fork target threadId must not be empty', ); } - return realize( + return await realize( targetSpec, target.driver, { recovery, forkInput: { source, - turns: readHistory(source.namespace, source.threadId, true), + turns: await readHistory(source.namespace, source.threadId, true), }, }, { driverState: target.driver.initialState() }, ); }, - rehome(source: ThreadIdentity, rawTargetSpec: WorkloadSpec): void { + async rehome( + source: ThreadIdentity, + rawTargetSpec: WorkloadSpec, + ): Promise { if (runtime.get(source.threadId) !== undefined) { throw new AgenetesError( 'rehome_conflict', `cannot rehome thread '${source.namespace.name}/${source.threadId}' with a live handle`, ); } - const sourceRecord = threadStore.get(source.namespace, source.threadId); + const sourceRecord = await threadStore.get( + source.namespace, + source.threadId, + ); if (!sourceRecord) { throw new AgenetesError( 'invalid_workload', @@ -679,13 +729,14 @@ export function createAgenetesInstance( ); } const targetHasRecord = - threadStore.get(targetSpec.namespace, targetSpec.threadId) !== + (await threadStore.get(targetSpec.namespace, targetSpec.threadId)) !== undefined; const targetHasTurns = - turnStore.list(targetSpec.namespace, targetSpec.threadId).length > 0; + (await turnStore.list(targetSpec.namespace, targetSpec.threadId)) + .length > 0; const targetHasEvents = - eventLog.readRecords(targetSpec.namespace, targetSpec.threadId).length > - 0; + (await eventLog.readRecords(targetSpec.namespace, targetSpec.threadId)) + .length > 0; if (targetHasRecord || targetHasTurns || targetHasEvents) { throw new AgenetesError( 'rehome_conflict', @@ -696,11 +747,14 @@ export function createAgenetesInstance( // Snapshot the complete source BEFORE any write, so a determinate // failure at any later step can restore it byte-for-byte regardless // of which step failed. - const sourceEvents = eventLog.readRecords( + const sourceEvents = await eventLog.readRecords( + source.namespace, + source.threadId, + ); + const sourceTurns = await turnStore.list( source.namespace, source.threadId, ); - const sourceTurns = turnStore.list(source.namespace, source.threadId); const targetRecord: ThreadRecord = { driverSchemaVersion: validatedSource.driverSchemaVersion, spec: targetSpec, @@ -710,9 +764,12 @@ export function createAgenetesInstance( // Each step's compensation is pushed ONLY once the step itself // durably succeeds, so a mid-sequence failure unwinds exactly the // completed prefix — never more, never less. - const undo: Array<() => void> = []; - const step = (write: () => void, compensate: () => void): void => { - write(); + const undo: Array<() => void | Promise> = []; + const step = async ( + write: () => void | Promise, + compensate: () => void | Promise, + ): Promise => { + await write(); undo.push(compensate); }; @@ -721,49 +778,65 @@ export function createAgenetesInstance( // thread record LAST — the record write is the destination // visibility point (I9.4): the first moment a reader can observe // the thread under `targetSpec.namespace`. - step( - () => - eventLog.replace( + await step( + async () => + await eventLog.replace( targetSpec.namespace, targetSpec.threadId, sourceEvents, ), - () => eventLog.delete(targetSpec.namespace, targetSpec.threadId), + async () => + await eventLog.delete(targetSpec.namespace, targetSpec.threadId), ); - step( - () => - turnStore.replace( + await step( + async () => + await turnStore.replace( targetSpec.namespace, targetSpec.threadId, sourceTurns, ), - () => turnStore.delete(targetSpec.namespace, targetSpec.threadId), + async () => + await turnStore.delete(targetSpec.namespace, targetSpec.threadId), ); - step( - () => - threadStore.upsert( + await step( + async () => + await threadStore.upsert( targetSpec.namespace, targetSpec.threadId, targetRecord, ), - () => threadStore.delete(targetSpec.namespace, targetSpec.threadId), + async () => + await threadStore.delete(targetSpec.namespace, targetSpec.threadId), ); // Only once the target is completely durable: remove the source // record (its own visibility point) before its now-orphaned logs. - step( - () => threadStore.delete(source.namespace, source.threadId), - () => - threadStore.upsert(source.namespace, source.threadId, sourceRecord), + await step( + async () => + await threadStore.delete(source.namespace, source.threadId), + async () => + await threadStore.upsert( + source.namespace, + source.threadId, + sourceRecord, + ), ); - step( - () => eventLog.delete(source.namespace, source.threadId), - () => - eventLog.replace(source.namespace, source.threadId, sourceEvents), + await step( + async () => await eventLog.delete(source.namespace, source.threadId), + async () => + await eventLog.replace( + source.namespace, + source.threadId, + sourceEvents, + ), ); - step( - () => turnStore.delete(source.namespace, source.threadId), - () => - turnStore.replace(source.namespace, source.threadId, sourceTurns), + await step( + async () => await turnStore.delete(source.namespace, source.threadId), + async () => + await turnStore.replace( + source.namespace, + source.threadId, + sourceTurns, + ), ); } catch (error) { // Unwind the completed prefix in reverse (LIFO) order, restoring @@ -776,7 +849,7 @@ export function createAgenetesInstance( const rollbackErrors: unknown[] = []; for (const compensate of undo.reverse()) { try { - compensate(); + await compensate(); } catch (rollbackError) { rollbackErrors.push(rollbackError); } @@ -794,7 +867,7 @@ export function createAgenetesInstance( get(threadId: string): AgentHandle | undefined { return runtime.get(threadId); }, - close(threadId: string): void { + async close(threadId: string): Promise { // Tear down the up-report listener + end any open notification streams // before evicting the live handle. const unsub = unsubscribers.get(threadId); @@ -802,40 +875,74 @@ export function createAgenetesInstance( unsub(); unsubscribers.delete(threadId); } - bus.closeThread(threadId); - runtime.close(threadId); + try { + await pendingReports.get(threadId); + } finally { + pendingReports.delete(threadId); + bus.closeThread(threadId); + runtime.close(threadId); + } }, - record(namespace: Namespace, threadId: string): ThreadRecord | undefined { - const record = threadStore.get(namespace, threadId); + async record( + namespace: Namespace, + threadId: string, + ): Promise { + await pendingReports.get(threadId); + const record = await threadStore.get(namespace, threadId); return record ? validateRecord(record) : undefined; }, - records(namespace: Namespace): ThreadRecord[] { - return threadStore.list(namespace).map(validateRecord); + async records(namespace: Namespace): Promise { + await Promise.all(pendingReports.values()); + return (await threadStore.list(namespace)).map(validateRecord); }, notifications(threadId: string): AsyncIterable { return bus.subscribe(threadId); }, - logMetadata(namespace: Namespace, threadId: string): ThreadLogMetadata { + async logMetadata( + namespace: Namespace, + threadId: string, + ): Promise { return { - eventCount: eventLog.maxSeq(namespace, threadId), - turnCount: turnStore.count(namespace, threadId), + eventCount: await eventLog.maxSeq(namespace, threadId), + turnCount: await turnStore.count(namespace, threadId), }; }, - history( + async history( namespace: Namespace, threadId: string, options?: HistoryOptions, - ): ThreadHistory { + ): Promise { return { - turns: readHistory(namespace, threadId, options?.withTail === true), + turns: await readHistory( + namespace, + threadId, + options?.withTail === true, + ), }; }, tail( namespace: Namespace, threadId: string, ): AsyncIterable { - const fence = turnStore.fence(namespace, threadId); - return createTail(eventLog, namespace, threadId, fence); + return createTail(eventLog, namespace, threadId, () => + turnStore.fence(namespace, threadId), + ); }, }; + // Async persistence adds scheduling points to the former synchronous + // lifecycle. Keep create/fork/rehome/close mutually exclusive so a move + // cannot race a spawn or a second move across its compensation boundary. + let lifecycle: Promise = Promise.resolve(); + const serialize = (operation: () => Promise): Promise => { + const result = lifecycle.catch(() => {}).then(operation); + lifecycle = result; + return result; + }; + return { + ...api, + create: (spec) => serialize(() => api.create(spec)), + fork: (source, target) => serialize(() => api.fork(source, target)), + rehome: (source, target) => serialize(() => api.rehome(source, target)), + close: (threadId) => serialize(() => api.close(threadId)), + }; } diff --git a/external/agenetes/packages/agenetes/src/mount.test.ts b/external/agenetes/packages/agenetes/src/mount.test.ts index 49255f7ec..aaeccc672 100644 --- a/external/agenetes/packages/agenetes/src/mount.test.ts +++ b/external/agenetes/packages/agenetes/src/mount.test.ts @@ -33,14 +33,15 @@ const driver = defineDriver({ describe('mountAgenetes static driver map', () => { it('mounts a complete host-constructed driver map', () => { const instance = mountAgenetes({ drivers: { external: driver } }); - expect(() => - instance.create({ - kind: 'external', - workloadType: 'Deployment', - namespace: { name: 'canvas-1' }, - threadId: 'thread-1', - spec: {}, - }), + expect( + async () => + await instance.create({ + kind: 'external', + workloadType: 'Deployment', + namespace: { name: 'canvas-1' }, + threadId: 'thread-1', + spec: {}, + }), ).not.toThrow(); }); diff --git a/external/agenetes/packages/agenetes/src/notifications.test.ts b/external/agenetes/packages/agenetes/src/notifications.test.ts index 6239c2355..b9147af0d 100644 --- a/external/agenetes/packages/agenetes/src/notifications.test.ts +++ b/external/agenetes/packages/agenetes/src/notifications.test.ts @@ -9,7 +9,11 @@ import { defineDriver } from '@agenetes/runtime'; import { describe, expect, it } from 'vitest'; -import { mountAgenetes } from './index.js'; +import { + mountAgenetes, + InMemoryThreadStore, + type ThreadStore, +} from './index.js'; import type { AgentSpec, @@ -113,14 +117,14 @@ describe('notification surface (M5.5/A3.0, I9.7)', () => { it('persists the up-reported snapshot then re-emits its metadata', async () => { const inst = mount((spec) => new ReportingHandle(spec)); const spec = deployment('thr_1'); - const handle = inst.create(spec) as unknown as ReportingHandle; + const handle = (await inst.create(spec)) as unknown as ReportingHandle; const collected = take(inst.notifications('thr_1'), 1); handle.emit({ driverState: { sessionId: 'sess-1' }, metadata: meta }); expect(await collected).toEqual([meta]); // persist-then-notify: the record already carries the full snapshot. - const rec = inst.record(spec.namespace, 'thr_1'); + const rec = await inst.record(spec.namespace, 'thr_1'); expect(rec).toMatchObject({ driverSchemaVersion: 1, state: { driverState: { sessionId: 'sess-1' }, metadata: meta }, @@ -130,7 +134,7 @@ describe('notification surface (M5.5/A3.0, I9.7)', () => { it('persists a driver-state-only snapshot without emitting to L1', async () => { const inst = mount((spec) => new ReportingHandle(spec)); const spec = deployment('thr_1'); - const handle = inst.create(spec) as unknown as ReportingHandle; + const handle = (await inst.create(spec)) as unknown as ReportingHandle; const collected = take(inst.notifications('thr_1'), 1); handle.emit({ driverState: { sessionId: 'sess-1' } }); @@ -140,46 +144,141 @@ describe('notification surface (M5.5/A3.0, I9.7)', () => { expect(await collected).toEqual([meta]); expect( ( - inst.record(spec.namespace, 'thr_1')?.state + (await inst.record(spec.namespace, 'thr_1'))?.state .driverState as StubDriverState ).sessionId, ).toBe('sess-1'); }); - it('wires the listener exactly once across get-or-create reuse', () => { + it('wires the listener exactly once across get-or-create reuse', async () => { const inst = mount((spec) => new ReportingHandle(spec)); const spec = deployment('thr_1'); - const h1 = inst.create(spec) as unknown as ReportingHandle; - const h2 = inst.create(spec) as unknown as ReportingHandle; + const h1 = (await inst.create(spec)) as unknown as ReportingHandle; + const h2 = (await inst.create(spec)) as unknown as ReportingHandle; expect(h1).toBe(h2); // reuse returns the same handle expect(h1.wired).toBe(true); }); it('close() ends every open notification stream', async () => { const inst = mount((spec) => new ReportingHandle(spec)); - inst.create(deployment('thr_1')); + await inst.create(deployment('thr_1')); const drained: AgentMetadata[] = []; const loop = (async () => { for await (const m of inst.notifications('thr_1')) drained.push(m); })(); - inst.close('thr_1'); + await inst.close('thr_1'); await loop; // returns because the stream ended expect(drained).toEqual([]); }); it('a handle with no onState leaves the notification stream empty', async () => { const inst = mount((spec) => new SilentHandle(spec)); - inst.create(deployment('thr_1')); + await inst.create(deployment('thr_1')); const drained: AgentMetadata[] = []; const loop = (async () => { for await (const m of inst.notifications('thr_1')) drained.push(m); })(); - inst.close('thr_1'); // nothing was ever published + await inst.close('thr_1'); // nothing was ever published await loop; expect(drained).toEqual([]); }); }); + +describe('asynchronous thread persistence', () => { + it('waits for initial persistence before exposing a live handle', async () => { + const backing = new InMemoryThreadStore(); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const store: ThreadStore = new Proxy(backing, { + get(target, key) { + if (key === 'upsert') + return async (...args: Parameters) => { + entered.resolve(); + await gate.promise; + target.upsert(...args); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + let created = 0; + const inst = mountAgenetes({ + drivers: { + external: driver((spec) => { + created++; + return new ReportingHandle(spec); + }), + }, + threadStore: store, + }); + const pending = inst.create(deployment('delayed')); + await entered.promise; + expect(created).toBe(0); + expect(inst.get('delayed')).toBeUndefined(); + gate.resolve(); + const handle = await pending; + expect(inst.get('delayed')).toBe(handle); + expect(created).toBe(1); + await inst.close('delayed'); + }); + + it.each([false, true])( + 'drains state writes on close and surfaces failure=%s', + async (fail) => { + const backing = new InMemoryThreadStore(); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + let reports = false; + const store: ThreadStore = new Proxy(backing, { + get(target, key) { + if (key === 'upsert') + return async (...args: Parameters) => { + if (reports) { + entered.resolve(); + await gate.promise; + if (fail) throw new Error('state write failed'); + } + target.upsert(...args); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const inst = mountAgenetes({ + drivers: { external: driver((spec) => new ReportingHandle(spec)) }, + threadStore: store, + }); + const spec = deployment('reported'); + const handle = (await inst.create(spec)) as unknown as ReportingHandle; + reports = true; + const seen: AgentMetadata[] = []; + const reading = (async () => { + for await (const value of inst.notifications('reported')) + seen.push(value); + })(); + handle.emit({ driverState: { sessionId: 'saved' }, metadata: meta }); + await entered.promise; + expect(seen).toEqual([]); + expect( + backing.get(spec.namespace, spec.threadId)?.state.driverState, + ).toEqual({}); + const closing = inst.close('reported'); + expect(handle.closed).toBe(false); + const checked = fail + ? expect(closing).rejects.toThrow('state write failed') + : closing; + gate.resolve(); + await checked; + await reading; + expect(handle.closed).toBe(true); + expect(seen).toEqual(fail ? [] : [meta]); + expect( + backing.get(spec.namespace, spec.threadId)?.state.driverState, + ).toEqual(fail ? {} : { sessionId: 'saved' }); + }, + ); +}); diff --git a/external/agenetes/packages/agenetes/src/thread-store.ts b/external/agenetes/packages/agenetes/src/thread-store.ts index 8360bad52..b3ae3d6e1 100644 --- a/external/agenetes/packages/agenetes/src/thread-store.ts +++ b/external/agenetes/packages/agenetes/src/thread-store.ts @@ -26,10 +26,17 @@ interface ThreadStoreFile { } export interface ThreadStore { - upsert(namespace: Namespace, threadId: string, record: ThreadRecord): void; - get(namespace: Namespace, threadId: string): ThreadRecord | undefined; - list(namespace: Namespace): ThreadRecord[]; - delete(namespace: Namespace, threadId: string): void; + upsert( + namespace: Namespace, + threadId: string, + record: ThreadRecord, + ): void | Promise; + get( + namespace: Namespace, + threadId: string, + ): ThreadRecord | undefined | Promise; + list(namespace: Namespace): ThreadRecord[] | Promise; + delete(namespace: Namespace, threadId: string): void | Promise; } export class InMemoryThreadStore implements ThreadStore { diff --git a/external/agenetes/packages/agenetes/src/turn-store.ts b/external/agenetes/packages/agenetes/src/turn-store.ts index 8c384a4a0..0807027c3 100644 --- a/external/agenetes/packages/agenetes/src/turn-store.ts +++ b/external/agenetes/packages/agenetes/src/turn-store.ts @@ -59,17 +59,20 @@ export interface TurnStore { namespace: Namespace, threadId: string, persisted: PersistedTurn, - ): void; + ): void | Promise; /** Read every folded turn for a thread, in fold (emission) order. */ - list(namespace: Namespace, threadId: string): PersistedTurn[]; + list( + namespace: Namespace, + threadId: string, + ): PersistedTurn[] | Promise; /** The number of folded turns persisted for a thread. */ - count(namespace: Namespace, threadId: string): number; + count(namespace: Namespace, threadId: string): number | Promise; /** * The `seqEnd` of the last folded turn — the Tier-1 fence a live tail * resumes from — or `0` when the thread has no folded turn yet (tail from * the very first event). */ - fence(namespace: Namespace, threadId: string): number; + fence(namespace: Namespace, threadId: string): number | Promise; /** * Overwrite a thread's ENTIRE Tier-2 log with `persisted` (already in fold * order), replacing whatever the target held before. A narrow capability @@ -81,13 +84,13 @@ export interface TurnStore { namespace: Namespace, threadId: string, persisted: readonly PersistedTurn[], - ): void; + ): void | Promise; /** * Remove a thread's Tier-2 log entirely (idempotent). Reserved for the * `rehome()` primitive: dropping the source log after its target twin is * durable, or compensating a target log written during a failed rehome. */ - delete(namespace: Namespace, threadId: string): void; + delete(namespace: Namespace, threadId: string): void | Promise; } /** Defensive shape-check for a persisted record read back from disk. */