diff --git a/packages/core/src/agents/goalSession/AuthoritativeGoalSessionRuntimePorts.ts b/packages/core/src/agents/goalSession/AuthoritativeGoalSessionRuntimePorts.ts new file mode 100644 index 000000000..07cb41a3d --- /dev/null +++ b/packages/core/src/agents/goalSession/AuthoritativeGoalSessionRuntimePorts.ts @@ -0,0 +1,148 @@ +import type { + GoalModelChangeHistoryPort, GoalProviderEffectStage, GoalProviderFirstEffectPort, + GoalProviderOperationFence, GoalSessionEventSink, GoalSessionMessagePort, + GoalSessionStatePort, GoalSessionTerminalPort, GoalSessionTransitionPort, + GoalStartedProviderEffect, +} from './contract.js'; +import type { GoalSessionRecoveryPort, GoalSessionRuntimePorts } from './runtimePorts.js'; +import { GoalSessionContractError } from './errors.js'; +import { + assertStartedProviderEffect, cleanupStartedProviderEffect, startedProviderEffectCleanup, +} from './providerEffectProtocol.js'; +import { assertGoalProviderEffectStage } from './providerOperationBoundary.js'; + +export type GoalProviderEffectClaimResult = + | { status: 'claimed' | 'recoverable'; token: string } + | { status: 'settled'; outcome: unknown } + | { status: 'terminal_in_doubt' }; + +/** + * Control-owned transaction hook. Implementations persist the stage claim before + * `runClaimedProviderEffect`, then revalidate owner/state/fence inside the same + * authoritative transaction that calls the synchronous callback and writes its + * started receipt. A duplicate or abandoned claim is in doubt and is never run. + */ +export interface GoalProviderEffectTransactionDomain { + claimProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + ): Promise; + runClaimedProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + token: string, + effect: () => GoalStartedProviderEffect, + ): Promise>; + settleProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + token: string, + outcome: unknown, + ): Promise; + poisonProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + token: string, + ): Promise; +} + +/** Ports supplied by the one authoritative migrated control repository. */ +export interface GoalSessionAuthoritativeTransactionDomain { + state: GoalSessionStatePort; + transitions: GoalSessionTransitionPort; + events: GoalSessionEventSink; + terminal: GoalSessionTerminalPort; + messages: GoalSessionMessagePort; + modelChanges: GoalModelChangeHistoryPort; + providerEffects: GoalProviderEffectTransactionDomain; +} + +/** + * Production composition adapter only: it owns no tables or SQLite connection. + * Missing control-domain injection is a construction error; there is no memory + * fallback. The injected domain is responsible for global session ownership. + */ +export class AuthoritativeGoalSessionRuntimePorts implements GoalProviderFirstEffectPort { + constructor( + private readonly domain: GoalSessionAuthoritativeTransactionDomain, + private readonly recovery: GoalSessionRecoveryPort, + ) { + if (!domain?.state || !domain.transitions || !domain.events || !domain.terminal + || !domain.messages || !domain.modelChanges || !domain.providerEffects + || typeof domain.providerEffects.claimProviderEffect !== 'function' + || typeof domain.providerEffects.runClaimedProviderEffect !== 'function' + || typeof domain.providerEffects.settleProviderEffect !== 'function' + || typeof domain.providerEffects.poisonProviderEffect !== 'function' + || typeof recovery?.inspectContainer !== 'function' || typeof recovery.inspectRepository !== 'function') { + throw new GoalSessionContractError( + 'Goal runtime requires an authoritative transaction domain', 'AUTHORITATIVE_DOMAIN_MISSING', + ); + } + } + + asRuntimePorts(): GoalSessionRuntimePorts { + return { + state: this.domain.state, + transitions: this.domain.transitions, + events: this.domain.events, + terminal: this.domain.terminal, + messages: this.domain.messages, + recovery: this.recovery, + modelChanges: this.domain.modelChanges, + providerFirstEffects: this, + }; + } + + async start( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + effect: () => GoalStartedProviderEffect, + rebuild: (value: T) => R, + ): Promise { + assertGoalProviderEffectStage(stage); + const claim = await this.domain.providerEffects.claimProviderEffect(fence, stage); + if (claim.status === 'settled') return rebuild(claim.outcome as T); + if (claim.status === 'terminal_in_doubt') throw new GoalSessionContractError( + 'Provider effect stage is already claimed and remains in doubt', 'PROVIDER_EFFECT_IN_DOUBT', + ); + let started: GoalStartedProviderEffect | undefined; + let committed: GoalStartedProviderEffect; + let cleanup: GoalStartedProviderEffect['cleanup'] | undefined; + try { + committed = await this.domain.providerEffects.runClaimedProviderEffect(fence, stage, claim.token, () => { + const candidate: unknown = effect(); + cleanup = startedProviderEffectCleanup(candidate); + assertStartedProviderEffect(candidate); + started = candidate; + return candidate; + }); + assertStartedProviderEffect(committed); + if (committed !== started) throw new GoalSessionContractError( + 'Authoritative domain returned a different started-effect handle', 'INVALID_FIRST_EFFECT_HANDLE', + ); + } catch (error) { + if (started || cleanup) { + try { + if (started) await cleanupStartedProviderEffect(started); + else await cleanup!.run(); + } + catch { + throw new GoalSessionContractError( + 'Started provider effect cleanup failed; durable stage remains in doubt', + 'PROVIDER_EFFECT_CLEANUP_FAILED', + ); + } + } + await this.domain.providerEffects.poisonProviderEffect(fence, stage, claim.token).catch(() => undefined); + throw error; + } + try { + const outcome = rebuild(await committed.completion); + await this.domain.providerEffects.settleProviderEffect(fence, stage, claim.token, outcome); + return outcome; + } catch (error) { + await this.domain.providerEffects.poisonProviderEffect(fence, stage, claim.token).catch(() => undefined); + throw error; + } + } +} diff --git a/packages/core/src/agents/goalSession/CodexAppServerOpen.ts b/packages/core/src/agents/goalSession/CodexAppServerOpen.ts new file mode 100644 index 000000000..d35d0ba6b --- /dev/null +++ b/packages/core/src/agents/goalSession/CodexAppServerOpen.ts @@ -0,0 +1,369 @@ +import type { + GoalProviderOpenContext, + GoalProviderSessionSnapshot, + GoalSessionJsonValue, +} from './contract.js'; +import { + CODEX_APP_SERVER_METHODS_0146, CODEX_APP_SERVER_PROTOCOL_0146, CODEX_CLI_VERSION_0146, + type CodexThreadResponse0146, type CodexThreadResumeParams0146, type CodexThreadStartParams0146, +} from './codexAppServer0146Bindings.generated.js'; +import { GoalSessionContractError, providerOpenInDoubtError } from './errors.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; +import { sanitizeNewRecoveryMetadata, sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +import { assertExactThreadFields, assertExactThreadResponseFields } from './codexAppServer0146Validation.js'; + +export const SUPERVISED_CODEX_MODEL = 'gpt-5.6-sol'; +export const SUPERVISED_CODEX_PROTOCOL = CODEX_APP_SERVER_PROTOCOL_0146; +const CODEX_CONTAINER_CWD = '/workspace'; +const MAX_APP_SERVER_LINE_BYTES = 1024 * 1024; +const MAX_REQUEST_BYTES = 2 * 1024 * 1024; +const MAX_MESSAGES_PER_REQUEST = 512; +const REQUEST_TIMEOUT_MS = 15_000; +const MAX_MODEL_PAGES = 16; +const MAX_MODELS = 1_600; + +type JsonObject = Record; + +/** + * Pinned Codex 0.146 stdio App Server lifecycle. The transport must already be + * constructed under the supervisor's durable open claim. No turn identity is + * accepted or invented by this control-scoped operation. + */ +export async function openSupervisedCodexAppServer( + context: GoalProviderOpenContext, + persisted?: GoalProviderSessionSnapshot, +): Promise { + validateContext(context); + const rpc = new StdioAppServerRpc(context); + let newThreadRequestStarted = false; + try { + const initialized = await rpc.request(CODEX_APP_SERVER_METHODS_0146.initialize, { + clientInfo: { + name: 'propr_goal_runtime', title: 'ProPR Goal Runtime', version: CODEX_CLI_VERSION_0146, + }, + capabilities: { experimentalApi: false, requestAttestation: false }, + }); + assertPinnedInitialize(initialized); + await rpc.notify(CODEX_APP_SERVER_METHODS_0146.initialized); + await probeExactModel(rpc); + + const persistedThread = decodePersistedThread(persisted, context); + let thread: JsonObject; + if (persistedThread) { + const params: CodexThreadResumeParams0146 = { + threadId: persistedThread.threadId, + model: SUPERVISED_CODEX_MODEL, + cwd: CODEX_CONTAINER_CWD, + approvalPolicy: 'never', + sandbox: 'workspace-write', + excludeTurns: true, + }; + thread = await rpc.request(CODEX_APP_SERVER_METHODS_0146.threadResume, params as unknown as JsonObject); + } else { + const params: CodexThreadStartParams0146 = { + model: SUPERVISED_CODEX_MODEL, + cwd: CODEX_CONTAINER_CWD, + approvalPolicy: 'never', + sandbox: 'workspace-write', + }; + newThreadRequestStarted = true; + thread = await rpc.request(CODEX_APP_SERVER_METHODS_0146.threadStart, params as JsonObject); + } + const identity = decodeThreadResponse(thread, persistedThread); + const recoveryMetadata = sanitizeNewRecoveryMetadata({ + version: 2, + provider: 'codex', + protocolVersion: SUPERVISED_CODEX_PROTOCOL, + payload: { + threadId: identity.threadId, + sessionId: identity.sessionId, + initialized: true, + checkpoint: persistedThread ? 'thread-resumed' : 'thread-started', + openKey: requiredOpenKey(context), + repository: context.repository.repository, + model: SUPERVISED_CODEX_MODEL, + providerHomeIdentity: context.providerHomeTarget, + cliVersion: CODEX_CLI_VERSION_0146, + }, + usage: { components: [] }, + }, 'codex'); + return { providerSessionId: identity.threadId, recoveryMetadata, model: SUPERVISED_CODEX_MODEL }; + } catch (error) { + if (newThreadRequestStarted && !persisted) throw providerOpenInDoubtError(); + if (error instanceof GoalSessionContractError) throw error; + throw new GoalSessionContractError('Codex App Server open failed safely', 'PROVIDER_OPERATION_FAILED'); + } finally { + await rpc.close().catch(() => undefined); + } +} + +class StdioAppServerRpc { + private readonly iterator: AsyncIterator; + private readonly lines: BoundedLineReader; + private requestSequence = 0; + private consumedBytes = 0; + + constructor(private readonly context: GoalProviderOpenContext) { + this.iterator = context.transport.output[Symbol.asyncIterator](); + this.lines = new BoundedLineReader(this.iterator); + } + + async close(): Promise { + try { this.context.transport.closeInput(); } + catch { /* Cancellation below remains mandatory. */ } + await this.context.transport.cancel(); + try { + const returned = this.iterator.return?.(); + if (returned) void returned.catch(() => undefined); + } catch { /* The owned process is already cancelled. */ } + } + + async notify(method: string): Promise { + await this.write({ method }); + } + + async request(method: string, params: JsonObject): Promise { + const id = `${this.context.executionId}-${this.context.attemptId}-${this.requestSequence}`; + this.requestSequence += 1; + return withTimeout(this.requestUntilResponse(method, id, params), this.context.transport); + } + + private async requestUntilResponse(method: string, id: string, params: JsonObject): Promise { + await this.write({ method, id, params }); + for (let count = 0; count < MAX_MESSAGES_PER_REQUEST; count += 1) { + const line = await this.lines.nextLine(); + this.consumedBytes += Buffer.byteLength(line); + if (this.consumedBytes > MAX_REQUEST_BYTES) throw new Error('App Server aggregate response is oversized'); + const message = parseMessage(line); + if (message.id !== id) continue; + if (message.error !== undefined) throw new Error('App Server rejected a request'); + return closedJsonObject(message.result, 'App Server response result'); + } + throw new Error('App Server response exceeded its message bound'); + } + + private async write(message: JsonObject): Promise { + const line = `${JSON.stringify(message)}\n`; + if (Buffer.byteLength(line) > MAX_APP_SERVER_LINE_BYTES) throw new Error('App Server request is oversized'); + await this.context.transport.write(line); + } +} + +class BoundedLineReader { + private readonly decoder = new TextDecoder('utf-8', { fatal: true }); + private buffered = ''; + + constructor(private readonly iterator: AsyncIterator) {} + + async nextLine(): Promise { + for (;;) { + const newline = this.buffered.indexOf('\n'); + if (newline >= 0) { + const line = this.buffered.slice(0, newline).replace(/\r$/, ''); + this.buffered = this.buffered.slice(newline + 1); + if (Buffer.byteLength(line) > MAX_APP_SERVER_LINE_BYTES) throw new Error('App Server line is oversized'); + if (!line) continue; + return line; + } + const next = await this.iterator.next(); + if (next.done) { + const tail = this.buffered; + this.buffered = ''; + if (tail) return tail; + throw new Error('App Server output ended before its response'); + } + if (typeof next.value !== 'string') throw new Error('App Server emitted a non-string protocol chunk'); + const bytes = Buffer.from(next.value); + this.buffered += this.decoder.decode(bytes, { stream: true }); + if (Buffer.byteLength(this.buffered) > MAX_APP_SERVER_LINE_BYTES) throw new Error('App Server line is oversized'); + // The production channel is newline-framed. Retain compatibility + // with an isolated transport that emits exactly one complete JSON + // value per iterator item without the delimiter. + if (isCompleteJsonObject(this.buffered)) { + const line = this.buffered; + this.buffered = ''; + return line; + } + } + } +} + +function isCompleteJsonObject(value: string): boolean { + if (!value.startsWith('{') || !value.endsWith('}')) return false; + try { return isObject(JSON.parse(value)); } + catch { return false; } +} + +async function withTimeout(operation: Promise, transport: GoalProviderOpenContext['transport']): Promise { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('App Server request timed out')), REQUEST_TIMEOUT_MS); + }); + try { + return await Promise.race([operation, timeout]); + } catch (error) { + await transport.cancel().catch(() => undefined); + throw error; + } finally { + if (timer) clearTimeout(timer); + void operation.catch(() => undefined); + } +} + +function decodePersistedThread( + persisted: GoalProviderSessionSnapshot | undefined, + context: GoalProviderOpenContext, +): { threadId: string; sessionId: string } | undefined { + if (!persisted) return undefined; + const metadata = sanitizeRecoveryMetadata(persisted.recoveryMetadata, 'codex'); + if (!isObject(metadata) || metadata.version !== 2 || !isObject(metadata.payload)) { + throw new Error('Codex recovery metadata is not a v2 envelope'); + } + const payload = metadata.payload; + const threadId = safeId(payload.threadId); + const sessionId = safeId(payload.sessionId); + if (persisted.providerSessionId !== threadId || persisted.model !== SUPERVISED_CODEX_MODEL + || payload.openKey !== requiredOpenKey(context) + || payload.repository !== context.repository.repository + || payload.model !== SUPERVISED_CODEX_MODEL + || payload.providerHomeIdentity !== context.providerHomeTarget + || payload.cliVersion !== CODEX_CLI_VERSION_0146 + || metadata.protocolVersion !== SUPERVISED_CODEX_PROTOCOL) { + throw new Error('Codex persisted resume identity does not match the exact open claim'); + } + return { threadId, sessionId }; +} + +function decodeThreadResponse( + result: JsonObject, + fallback?: { threadId: string; sessionId: string }, +): { threadId: string; sessionId: string } { + const response = exactJsonObject(result, [ + 'thread', 'model', 'modelProvider', 'serviceTier', 'cwd', 'runtimeWorkspaceRoots', + 'instructionSources', 'approvalPolicy', 'approvalsReviewer', 'sandbox', + 'activePermissionProfile', 'reasoningEffort', 'multiAgentMode', + ...(fallback ? ['initialTurnsPage', 'turnsBackwardsCursor', 'itemsBackwardsCursor'] : []), + ], 'App Server thread response') as unknown as CodexThreadResponse0146; + assertExactThreadResponseFields(response); + if (response.model !== SUPERVISED_CODEX_MODEL || response.cwd !== CODEX_CONTAINER_CWD) { + throw new Error('App Server ignored or rerouted the exact model or workspace'); + } + const thread = decodeExactThread(response.thread); + const threadId = safeId(thread.id); + const sessionId = safeId(thread.sessionId); + if (fallback && (fallback.threadId !== threadId || fallback.sessionId !== sessionId)) { + throw new Error('App Server resumed a different thread identity'); + } + return { threadId, sessionId }; +} + +function decodeExactThread(value: unknown): JsonObject { + const thread = exactJsonObject(value, [ + 'id', 'extra', 'sessionId', 'forkedFromId', 'parentThreadId', 'preview', 'ephemeral', 'isPinned', + 'historyMode', 'modelProvider', 'createdAt', 'updatedAt', 'recencyAt', 'status', 'path', 'cwd', + 'cliVersion', 'source', 'canAcceptDirectInput', 'threadSource', 'agentNickname', 'agentRole', + 'gitInfo', 'name', 'turns', + ], 'App Server thread'); + assertExactThreadFields(thread as unknown as import('./codexAppServer0146Bindings.generated.js').CodexThread0146); + if (thread.cwd !== CODEX_CONTAINER_CWD || thread.cliVersion !== CODEX_CLI_VERSION_0146) { + throw new Error('App Server thread identity is not from exact Codex 0.146 App Server'); + } + return thread; +} + +function assertPinnedInitialize(result: JsonObject): void { + for (const field of ['userAgent', 'codexHome', 'platformFamily', 'platformOs']) { + if (typeof result[field] !== 'string' || !result[field]) throw new Error('App Server initialize response is malformed'); + } + if (result.codexHome !== '/home/node/.codex' || result.platformOs !== 'linux') { + throw new Error('App Server initialize identity is not the supervised container'); + } + if (typeof result.userAgent !== 'string' + || !result.userAgent.startsWith(`propr_goal_runtime/${CODEX_CLI_VERSION_0146} (`)) { + throw new Error('App Server CLI version is not exactly pinned'); + } +} + +async function probeExactModel(rpc: StdioAppServerRpc): Promise { + let cursor: string | null = null; + let supported = false; + let total = 0; + const seen = new Set(); + for (let page = 0; page < MAX_MODEL_PAGES; page += 1) { + const raw = await rpc.request(CODEX_APP_SERVER_METHODS_0146.modelList, { + limit: 100, includeHidden: true, cursor, + }); + const result = exactJsonObject(raw, ['data', 'nextCursor'], 'App Server model/list response'); + if (!Array.isArray(result.data) || result.data.length > 100 + || (result.nextCursor !== null && typeof result.nextCursor !== 'string')) { + throw new Error('App Server model probe is malformed'); + } + total += result.data.length; + if (total > MAX_MODELS) throw new Error('App Server model probe exceeded its aggregate bound'); + supported ||= result.data.some(value => { + const model = closedJsonObject(value, 'App Server model'); + return model.model === SUPERVISED_CODEX_MODEL || model.id === SUPERVISED_CODEX_MODEL; + }); + if (result.nextCursor === null) break; + cursor = result.nextCursor; + if (seen.has(cursor)) throw new Error('App Server model pagination cursor repeated'); + seen.add(cursor); + if (page === MAX_MODEL_PAGES - 1) throw new Error('App Server model pagination exceeded its page bound'); + } + if (!supported) throw new Error('App Server does not support exact gpt-5.6-sol'); +} + +function requiredOpenKey(context: GoalProviderOpenContext): string { + return safeId(context.deterministicOpenKey); +} + +function parseMessage(line: string): JsonObject { + if (typeof line !== 'string' || Buffer.byteLength(line) > MAX_APP_SERVER_LINE_BYTES) throw new Error('App Server line is invalid'); + let value: unknown; + try { value = JSON.parse(line); } + catch { throw new Error('App Server emitted invalid JSON'); } + return closedJsonObject(value, 'App Server message'); +} + +function validateContext(context: GoalProviderOpenContext): void { + if (context.requestedModel !== SUPERVISED_CODEX_MODEL) { + throw new GoalSessionContractError('Supervised Codex open requires exact gpt-5.6-sol', 'MODEL_ACK_MISMATCH'); + } + if (!context.repository.worktreePath.startsWith('/') || context.providerHomeTarget !== '/home/node/.codex') { + throw new GoalSessionContractError('Codex open context is not canonical', 'UNSAFE_PROVIDER_VALUE'); + } + requiredOpenKey(context); +} + +function safeId(value: GoalSessionJsonValue | undefined): string { + if (!isSafeIdentifier(value)) throw new Error('App Server identity is invalid'); + return value; +} + +function closedJsonObject(value: unknown, name: string): JsonObject { + if (!isObject(value)) throw new Error(`${name} is malformed`); + // JSON.parse results are data-only, but callers can also pass hostile + // persisted/test objects. Rebuild recursively through serialization after + // checking for accessors and non-data prototypes. + const descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.getOwnPropertySymbols(value).length + || Object.values(descriptors).some(descriptor => !descriptor.enumerable || !('value' in descriptor))) { + throw new Error(`${name} contains an accessor`); + } + return JSON.parse(JSON.stringify(value)) as JsonObject; +} + +function exactJsonObject(value: unknown, fields: readonly string[], name: string): JsonObject { + const result = closedJsonObject(value, name); + const actual = Object.keys(result); + if (actual.length !== fields.length || fields.some(field => !(field in result))) { + throw new Error(`${name} does not match the generated Codex 0.146 schema`); + } + return result; +} + +function isObject(value: unknown): value is JsonObject { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} diff --git a/packages/core/src/agents/goalSession/DockerGoalSessionRecovery.ts b/packages/core/src/agents/goalSession/DockerGoalSessionRecovery.ts new file mode 100644 index 000000000..0ecf70038 --- /dev/null +++ b/packages/core/src/agents/goalSession/DockerGoalSessionRecovery.ts @@ -0,0 +1,155 @@ +import { execFile } from 'node:child_process'; +import { access, realpath } from 'node:fs/promises'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import type { + GoalContainerInspection, + GoalRepositoryIdentity, + GoalRepositoryInspection, + GoalSessionIdentity, + GoalSessionRecoveryPort, +} from './contract.js'; +import { + fingerprintGoalWorktree, + normalizeGitRepositoryIdentity, + normalizeGoalRepositoryIdentity, + isSensitiveWorktreePath, +} from './worktreeIdentity.js'; + +const execFileAsync = promisify(execFile); + +/** Read-only Docker/worktree inspection used during daemon or worker restart reconciliation. */ +export class DockerGoalSessionRecovery implements GoalSessionRecoveryPort { + constructor( + private readonly dockerPath = '/usr/bin/docker', + private readonly gitPath = '/usr/bin/git', + ) {} + + async inspectContainer(identity: GoalSessionIdentity): Promise { + try { + const { stdout } = await execFileAsync(this.dockerPath, [ + 'ps', '-a', + '--filter', `label=propr.goal.id=${identity.goalId}`, + '--filter', `label=propr.goal.session=${identity.sessionId}`, + '--format', '{{.ID}}\t{{.Names}}\t{{.State}}', + ], { timeout: 10_000 }); + const records = stdout.trim().split('\n').filter(Boolean); + if (records.length === 0) return { status: 'missing', reason: 'No container has the persisted goal/session labels' }; + if (records.length > 1) { + return { status: 'daemon_unavailable', reason: 'Multiple containers claim the same goal session; manual cleanup is required' }; + } + const [containerId, containerName, rawState] = records[0].split('\t'); + const status = rawState === 'running' || rawState === 'restarting' ? 'running' : 'exited'; + const { stdout: labelOutput } = await execFileAsync(this.dockerPath, [ + 'inspect', '--format', '{{json .Config.Labels}}', containerId, + ], { timeout: 10_000 }); + const labels = JSON.parse(labelOutput) as Record; + const executionEpoch = Number(labels['propr.goal.controller-epoch']); + const hasIdentity = labels['propr.goal.id'] && labels['propr.goal.session'] + && labels['propr.goal.turn'] && labels['propr.goal.attempt'] + && labels['propr.goal.worktree-fingerprint'] + && Number.isSafeInteger(executionEpoch) && executionEpoch >= 0; + return { + status, + containerId, + containerName, + recoveryIdentity: hasIdentity ? { + goalId: labels['propr.goal.id'], + sessionId: labels['propr.goal.session'], + executionEpoch, + turnId: labels['propr.goal.turn'], + attemptId: labels['propr.goal.attempt'], + worktreeFingerprint: labels['propr.goal.worktree-fingerprint'], + } : undefined, + reason: hasIdentity + ? `Docker reports container state ${rawState || 'unknown'}` + : 'Recovered container is missing one or more authoritative identity labels', + }; + } catch (error) { + void error; + return { status: 'daemon_unavailable', reason: 'Docker inspection failed safely' }; + } + } + + async inspectRepository(repository: GoalRepositoryIdentity): Promise { + const safeRepository = normalizeGoalRepositoryIdentity(repository); + if (!safeRepository) { + return { + repository: '', + worktreePath: '/invalid-goal-worktree', + branch: 'invalid', + exists: false, + reason: 'Git remote does not contain a trustworthy repository identity', + }; + } + try { + await access(safeRepository.worktreePath); + } catch (error) { + void error; + return { ...safeRepository, exists: false, reason: 'Worktree is unavailable' }; + } + try { + const lexicalPath = path.resolve(safeRepository.worktreePath); + const resolvedWorktreePath = await realpath(safeRepository.worktreePath); + if (isSensitiveWorktreePath(resolvedWorktreePath)) { + return { + ...safeRepository, exists: false, + reason: 'Worktree is not an eligible project checkout', + }; + } + if (resolvedWorktreePath !== lexicalPath) { + return { + ...safeRepository, + exists: true, + resolvedWorktreePath, + reason: 'Worktree path resolves through a symlink or alias', + }; + } + const [{ stdout: head }, { stdout: status }, { stdout: branch }, { stdout: remote }, { stdout: root }] = await Promise.all([ + execFileAsync(this.gitPath, ['rev-parse', 'HEAD'], { cwd: safeRepository.worktreePath, timeout: 10_000 }), + execFileAsync(this.gitPath, ['status', '--porcelain'], { cwd: safeRepository.worktreePath, timeout: 10_000 }), + execFileAsync(this.gitPath, ['rev-parse', '--abbrev-ref', 'HEAD'], { cwd: safeRepository.worktreePath, timeout: 10_000 }), + execFileAsync(this.gitPath, ['config', '--get', 'remote.origin.url'], { cwd: safeRepository.worktreePath, timeout: 10_000 }), + execFileAsync(this.gitPath, ['rev-parse', '--show-toplevel'], { cwd: safeRepository.worktreePath, timeout: 10_000 }), + ]); + const observedRepository = normalizeGitRepositoryIdentity(remote); + if (!observedRepository) { + return { + ...safeRepository, + exists: true, + resolvedWorktreePath, + reason: 'Git remote does not contain a trustworthy repository identity', + }; + } + const observedBranch = branch.trim(); + if (path.resolve(root.trim()) !== resolvedWorktreePath) { + return { + ...safeRepository, + exists: true, + resolvedWorktreePath, + reason: 'Worktree path is not the observed Git repository root', + }; + } + return { + ...safeRepository, + exists: true, + dirty: Boolean(status.trim()), + observedRepository, + observedHeadSha: head.trim(), + observedBranch, + observedWorktreeFingerprint: fingerprintGoalWorktree({ + repository: observedRepository, + worktreePath: resolvedWorktreePath, + branch: observedBranch, + }), + resolvedWorktreePath, + }; + } catch { + return { + ...safeRepository, + exists: true, + reason: 'External worktree state could not be inspected safely', + }; + } + } +} diff --git a/packages/core/src/agents/goalSession/GoalCancellationControls.ts b/packages/core/src/agents/goalSession/GoalCancellationControls.ts new file mode 100644 index 000000000..c6494abb7 --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalCancellationControls.ts @@ -0,0 +1,261 @@ +import type { + GoalCancelRequest, GoalPendingCancellationContext, GoalProviderCancelRequest, + GoalSessionControlFence, GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { GoalImmediateModelControls } from './GoalImmediateModelControls.js'; +import { safeFailureDiagnostic } from './securityBoundary.js'; +import { nextState, persistedSnapshot } from './support.js'; +import { rebuildVoidProviderResult } from './providerResultBoundary.js'; + +/** Durable two-phase provider invalidation and idempotent cancellation replay. */ +export abstract class GoalCancellationControls extends GoalImmediateModelControls { + async cancel(request: GoalCancelRequest): Promise { + let state = await this.claimCancellation(request); + if (state.status === 'terminated' || state.status === 'failed') { + if (state.providerBarrierIntent?.phase === 'pending') { + state = await this.repairPendingProviderBarrier(request, state); + } + return state; + } + return this.resumeClaimedCancellation(request, state); + } + + protected async resumeClaimedCancellation( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise { + if (state.status === 'terminated') return state; + if (state.status !== 'cancelling' || !state.cancellationIntent) { + throw new GoalSessionContractError('Cancelling state is missing its durable cancellation intent', 'CANCELLATION_INTENT_MISSING'); + } + const intent = state.cancellationIntent; + const request: GoalProviderCancelRequest = { + goalId: fence.goalId, sessionId: fence.sessionId, controllerEpoch: fence.controllerEpoch, + reason: safeFailureDiagnostic(intent.reason, 'Operator cancelled the goal session'), + cancellationId: intent.cancellationId, + operationGeneration: state.providerOperationGeneration ?? 0, + operationFence: this.providerOperationFence( + fence, state.providerOperationGeneration ?? 0, + { kind: 'cancel', operationId: intent.cancellationId }, + ), + }; + const signalError = await this.signalProviderCancellation(fence, state, request); + const completion = await this.completeClaimedCancellation(fence, state, request); + state = completion.state; + await this.publishProviderOperationBarrier( + fence, state.providerOperationGeneration ?? request.operationGeneration, intent.cancellationId, + ); + state = await this.markBarrierPublished(fence, state); + if (completion.won && signalError && !(signalError instanceof CancellationTimedOut) + && !isDurablyClaimedCancellation(signalError)) throw signalError; + return state; + } + + private async signalProviderCancellation( + fence: GoalSessionControlFence, + state: GoalSessionState, + request: GoalProviderCancelRequest, + ): Promise { + const intent = state.cancellationIntent!; + try { + await this.publishProviderOperationBarrier(fence, request.operationGeneration, intent.cancellationId); + const authoritative = await this.requireControlledStateForBarrier(fence); + assertCancellationAuthority(authoritative, request); + const signal = this.providerFirstEffect(request.operationFence, () => { + const completion = intent.pendingContext + ? this.adapter.cancelPending!(request, intent.pendingContext) + : this.adapter.cancel(request, persistedSnapshot(state)); + // Cancellation is itself the ownership release. On a transaction + // failure, settling this idempotent cancellation completes cleanup. + return this.startedProviderEffect(completion, async () => { await completion; }); + }, rebuildVoidProviderResult); + await boundedCancellation(signal); + return undefined; + } catch (error) { + return error; + } + } + + private async completeClaimedCancellation( + fence: GoalSessionControlFence, + state: GoalSessionState, + request: GoalProviderCancelRequest, + ): Promise<{ state: GoalSessionState; won: boolean }> { + const intent = state.cancellationIntent!; + try { + state = await this.commitControlCompletion(state, request, { + status: 'terminated', activeTurn: undefined, initializationIntent: undefined, + retryTurn: undefined, recoveryAttempt: undefined, completedRecovery: undefined, + resumeIntent: undefined, completedResume: undefined, + providerOperationGeneration: (state.providerOperationGeneration ?? 0) + 1, + providerBarrierIntent: { + generation: (state.providerOperationGeneration ?? 0) + 1, + operationId: `${intent.cancellationId}:terminal`, kind: 'terminal', phase: 'pending', + claimedAt: new Date().toISOString(), pendingCancellationId: intent.cancellationId, + }, + pendingAfterTurnPause: undefined, modelChangeIntent: undefined, modelChangeIntents: undefined, + }, { type: 'completion', outcome: 'cancelled', error: intent.reason }); + return { state, won: true }; + } catch (error) { + if (!(error instanceof StaleGoalSessionFenceError)) throw error; + const current = await this.requireState(fence); + if (current.status !== 'terminated' + || current.cancellationIntent?.cancellationId !== intent.cancellationId) throw error; + state = await this.repairPendingProviderBarrier({ + ...fence, controllerEpoch: current.controllerEpoch, + }, current); + return { state, won: false }; + } + } + + protected async repairPendingProviderBarrier( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise { + const barrier = state.providerBarrierIntent; + if (!barrier || barrier.phase === 'published') return state; + if (barrier.kind === 'cancellation') { + return this.publishAndFinalizeCancellationBarrier({ + ...fence, reason: state.cancellationIntent?.reason ?? 'Resume durable cancellation', + }, state); + } + await this.publishProviderOperationBarrier(fence, barrier.generation, barrier.pendingCancellationId); + return this.markBarrierPublished(fence, await this.requireControlledStateForBarrier(fence)); + } + + private async claimCancellation(request: GoalCancelRequest): Promise { + for (;;) { + let state = await this.requireControlledStateForBarrier(request); + if (state.status === 'terminated' || state.status === 'failed') return state; + if (state.providerBarrierIntent?.phase === 'pending') { + if (state.providerBarrierIntent.kind !== 'cancellation' || !state.cancellationIntent) { + throw new GoalSessionContractError('A different provider invalidation is pending', 'PROVIDER_BARRIER_PENDING'); + } + state = await this.publishAndFinalizeCancellationBarrier(request, state); + return state; + } + if (state.status === 'cancelling' && state.cancellationIntent) return state; + if (!state.providerSessionId && (!state.initializationIntent || !this.adapter.cancelPending)) { + throw new GoalSessionContractError( + 'A lazy-ID provider must implement pending cancellation before it can be cancelled safely', + 'CAPABILITY_METHOD_MISSING', + ); + } + const pendingContext = this.pendingCancellationContext(state); + const reason = safeFailureDiagnostic(request.reason, 'Operator cancelled the goal session'); + const cancellationId = this.controlOperationId('cancel', state); + const generation = (state.providerOperationGeneration ?? 0) + 1; + const claimed = await this.ports.state.compareAndSet(state, nextState(state, { + providerOperationGeneration: generation, + cancellationIntent: { cancellationId, reason, claimedAt: new Date().toISOString(), pendingContext }, + // The invalidation claim itself removes every local mutation + // authority. Publication may block forever; output, message + // acknowledgement, completion, resume, recovery and control + // transitions are already impossible in the durable record. + status: 'cancelling', + activeTurn: undefined, + recoveryAttempt: undefined, + completedRecovery: undefined, + resumeIntent: undefined, + completedResume: undefined, + providerBarrierIntent: { + generation, operationId: cancellationId, kind: 'cancellation', phase: 'pending', + claimedAt: new Date().toISOString(), pendingCancellationId: cancellationId, + }, + })); + if (claimed) return this.publishAndFinalizeCancellationBarrier(request, claimed); + } + } + + private async publishAndFinalizeCancellationBarrier( + request: GoalCancelRequest, + state: GoalSessionState, + ): Promise { + const barrier = state.providerBarrierIntent; + const cancellation = state.cancellationIntent; + if (!barrier || barrier.kind !== 'cancellation' || !cancellation) { + throw new GoalSessionContractError('Cancellation barrier identity is missing', 'CANCELLATION_INTENT_MISSING'); + } + await this.publishProviderOperationBarrier(request, barrier.generation, cancellation.cancellationId); + const current = await this.requireControlledStateForBarrier(request); + if (current.providerBarrierIntent?.operationId !== barrier.operationId + || current.cancellationIntent?.cancellationId !== cancellation.cancellationId) { + throw new StaleGoalSessionFenceError('Cancellation barrier was replaced during publication'); + } + return this.compareAndSetExact(current, { + providerBarrierIntent: { ...barrier, phase: 'published' }, + }, 'A newer operation superseded cancellation publication'); + } + + private async markBarrierPublished( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise { + const barrier = state.providerBarrierIntent; + if (!barrier || barrier.phase === 'published') return state; + const saved = await this.ports.state.compareAndSet(state, nextState(state, { + providerBarrierIntent: { ...barrier, phase: 'published' }, + })); + if (saved) return saved; + const current = await this.requireControlledStateForBarrier(fence); + if (current.providerBarrierIntent?.operationId === barrier.operationId + && current.providerBarrierIntent.phase === 'published') return current; + throw new StaleGoalSessionFenceError('Provider barrier publication lost its durable identity'); + } + + private pendingCancellationContext(state: GoalSessionState): GoalPendingCancellationContext | undefined { + if (state.providerSessionId) return undefined; + if (!state.initializationIntent || !this.adapter.cancelPending) { + throw new GoalSessionContractError( + 'A lazy-ID provider must implement pending cancellation before it can be cancelled safely', + 'CAPABILITY_METHOD_MISSING', + ); + } + return { + initializationIntent: { + attemptId: state.initializationIntent.attemptId, + deterministicOpenKey: state.initializationIntent.deterministicOpenKey, + recordedAt: state.initializationIntent.recordedAt, + }, + activeTurn: state.activeTurn ? { + turnId: state.activeTurn.turnId, + executionId: state.activeTurn.executionId, + attemptId: state.activeTurn.attemptId, + } : undefined, + }; + } +} + +function isDurablyClaimedCancellation(error: unknown): boolean { + return error instanceof StaleGoalSessionFenceError + || error instanceof GoalSessionContractError && error.code === 'PROVIDER_EFFECT_IN_DOUBT'; +} + +const CANCELLATION_TIMEOUT_MS = 1_000; +class CancellationTimedOut extends Error {} + +async function boundedCancellation(signal: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new CancellationTimedOut('Provider cancellation timed out')), CANCELLATION_TIMEOUT_MS); + }); + try { + await Promise.race([signal, timeout]); + } finally { + if (timer) clearTimeout(timer); + void signal.catch(() => undefined); + } +} + +function assertCancellationAuthority( + state: GoalSessionState, + request: GoalProviderCancelRequest, +): void { + if (state.status !== 'cancelling' + || state.providerOperationGeneration !== request.operationGeneration + || state.cancellationIntent?.cancellationId !== request.cancellationId + || state.providerBarrierIntent?.phase !== 'published') { + throw new StaleGoalSessionFenceError('Provider cancellation was durably replaced'); + } +} diff --git a/packages/core/src/agents/goalSession/GoalContainerSupervisor.ts b/packages/core/src/agents/goalSession/GoalContainerSupervisor.ts new file mode 100644 index 000000000..457cf8fb2 --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalContainerSupervisor.ts @@ -0,0 +1,483 @@ +import { appendFile, mkdir, realpath, stat } from 'node:fs/promises'; +import path from 'node:path'; +import { + executeSupervisedDockerCommand, + type SupervisedDockerExecution, + type SupervisedDockerOutput, +} from '../../claude/docker/dockerExecutor.js'; +import type { + GoalExecutionIdentity, + GoalProviderFirstEffectPort, + GoalSessionEventSink, + GoalSessionFence, +} from './contract.js'; +import type { GoalSupervisedOpenClaim } from './goalSessionOpen.js'; +import { PendingOpenOwnership } from './pendingOpenOwnership.js'; +import { cleanTerminalGoalSession } from './terminalContainerCleanup.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { startedProviderEffect } from './providerEffectProtocol.js'; +import { assertSafeCallerTurnIdentity } from './safeIdentifier.js'; +import { sanitizeGoalSessionEvent } from './securityBoundary.js'; +import { isSensitiveHostSourcePath } from './worktreeIdentity.js'; +import { + buildGoalContainerLayout, buildGoalOpenContainerLayout, DEFAULT_GOAL_CONTAINER_RETENTION, + validateAbsolutePath, validateBindMountPath, + type GoalContainerIsolationPolicy, type GoalContainerLayout, type GoalContainerRetentionPolicy, + type GoalCredentialMount, type StartGoalContainerRequest, type StartGoalOpenContainerRequest, +} from './goalContainerLayout.js'; +export { + buildGoalContainerLayout, buildGoalOpenContainerLayout, DEFAULT_GOAL_CONTAINER_RETENTION, +} from './goalContainerLayout.js'; +export type { + GoalContainerIsolationPolicy, GoalContainerLayout, GoalContainerOutputObserver, + GoalContainerRetentionPolicy, GoalCredentialMount, StartGoalContainerRequest, StartGoalOpenContainerRequest, +} from './goalContainerLayout.js'; + +export interface GoalContainerSupervisorOptions { + isolation?: GoalContainerIsolationPolicy; + providerFirstEffects?: GoalProviderFirstEffectPort; + dockerPath?: string; +} + +function resolveSupervisorOptions( + value: GoalContainerSupervisorOptions | GoalContainerIsolationPolicy, +): GoalContainerSupervisorOptions { + return 'environmentKeys' in value && 'worktreePaths' in value && 'providerHomeTargets' in value + ? { isolation: value } : value; +} + +/** Container paths a provider home may never shadow. */ +const RESERVED_CONTAINER_PATHS = new Set(['/', '/workspace', '/etc', '/root', '/home', '/usr', '/bin', '/var', '/tmp', '/proc', '/sys', '/dev']); +/** Provider homes must live under one of these provider-owned roots. */ +const PROVIDER_HOME_ROOTS = ['/home/', '/root/', '/opt/']; +const CREDENTIAL_TARGET_DENY_TREES = ['/proc', '/sys', '/dev']; +const MAX_GOAL_LOG_BYTES = 8 * 1024 * 1024; + +const BLOCKED_ENVIRONMENT_KEYS = /^(?:DOCKER(?:_|$)|LD_PRELOAD$|LD_LIBRARY_PATH$|SSH(?:_|$)|HOME$|NODE_OPTIONS$|GIT_ASKPASS$|GIT_SSH(?:_|$)|AWS_(?:CONFIG|SHARED_CREDENTIALS)_FILE$|GOOGLE_APPLICATION_CREDENTIALS$)/; + +function validateEnvironment(environment: Record, allowedKeys: ReadonlySet): void { + for (const name of Object.keys(environment)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) throw new Error(`Invalid container environment name: ${name}`); + if (BLOCKED_ENVIRONMENT_KEYS.test(name.toUpperCase())) { + throw new Error(`Container environment key ${name} is host-controlled or sensitive and may not be forwarded`); + } + if (!allowedKeys.has(name)) throw new Error(`Container environment key ${name} is not explicitly allow-listed`); + } +} + +/** Rejects a provider home that would shadow /workspace, /, or another sensitive mount. */ +function validateProviderHomeTarget(target: string, allowedTargets: ReadonlySet): void { + validateBindMountPath(target, 'Provider home target'); + const normalized = path.posix.normalize(target).replace(/\/+$/, '') || '/'; + if (target !== normalized) throw new Error('Provider home target must be canonical and may not contain traversal aliases'); + if (RESERVED_CONTAINER_PATHS.has(normalized)) { + throw new Error(`Provider home target may not shadow the reserved container path ${normalized}`); + } + if (normalized === '/workspace' || normalized.startsWith('/workspace/')) { + throw new Error('Provider home target may not be placed inside the workspace mount'); + } + if (!PROVIDER_HOME_ROOTS.some(root => normalized.startsWith(root))) { + throw new Error(`Provider home target must be under a provider-owned root (${PROVIDER_HOME_ROOTS.join(', ')})`); + } + if (!allowedTargets.has(normalized)) throw new Error(`Provider home target ${normalized} is not explicitly allow-listed`); +} + +const SENSITIVE_SOURCE_SEGMENT = /(?:^|\/)(?:\.ssh|\.aws|\.docker|\.config|id_rsa|id_ed25519)(?:\/|$)/i; +const CONTAINER_SOCKET_PATHS = new Set(['/var/run/docker.sock', '/run/docker.sock', '/run/podman/podman.sock']); +const CREDENTIAL_SOURCE_DENY_TREES = ['/proc', '/sys', '/dev', '/run', '/etc', '/boot', '/bin', '/sbin', '/usr', '/var/lib/docker', '/var/lib/containers']; + +async function resolveApprovedSource(source: string, allowedSources: ReadonlySet, name: string): Promise { + validateBindMountPath(source, name); + const lexical = path.resolve(source); + if (source !== lexical) throw new Error(`${name} must be canonical and may not contain traversal aliases`); + if (name === 'Goal worktree path' && isSensitiveHostSourcePath(lexical)) { + throw new Error(`${name} may not be a sensitive host root or descendant`); + } + const resolved = await realpath(lexical).catch(() => null); + if (!resolved || resolved !== lexical) throw new Error(`${name} must exist and may not use a symlink alias`); + if (name === 'Goal worktree path' && isSensitiveHostSourcePath(resolved)) { + throw new Error(`${name} may not resolve to a sensitive host root or descendant`); + } + if (!allowedSources.has(resolved)) throw new Error(`${name} is not explicitly allow-listed`); + return resolved; +} + +async function canonicalCredentialSource(source: string): Promise { + validateBindMountPath(source, 'Credential mount source'); + const lexical = path.resolve(source); + if (source !== lexical) throw new Error('Credential mount source must be canonical and may not contain traversal aliases'); + const resolved = await realpath(lexical).catch(() => null); + if (!resolved || resolved !== lexical) throw new Error('Credential mount source must exist and may not use a symlink alias'); + if (CONTAINER_SOCKET_PATHS.has(lexical) || CONTAINER_SOCKET_PATHS.has(resolved) + || SENSITIVE_SOURCE_SEGMENT.test(resolved) + || CREDENTIAL_SOURCE_DENY_TREES.some(root => resolved === root || resolved.startsWith(`${root}/`))) { + throw new Error('Credential mount source is a broad or sensitive host path'); + } + if (!(await stat(resolved)).isFile()) throw new Error('Credential mount source is a broad or sensitive path, not a regular file'); + return resolved; +} + +function canonicalCredentialTarget(target: string): string { + validateBindMountPath(target, 'Credential mount target'); + const normalized = path.posix.normalize(target).replace(/\/+$/, ''); + if (target !== normalized) throw new Error('Credential mount target must be canonical and may not contain traversal aliases'); + if (RESERVED_CONTAINER_PATHS.has(normalized) || CONTAINER_SOCKET_PATHS.has(normalized) + || CREDENTIAL_TARGET_DENY_TREES.some(root => normalized.startsWith(`${root}/`)) + || normalized.startsWith('/etc/')) { + throw new Error('Credential mount target is a broad or sensitive container path'); + } + return normalized; +} + +function utf8Prefix(value: string, maxBytes: number): string { + if (Buffer.byteLength(value) <= maxBytes) return value; + let low = 0; + let high = value.length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (Buffer.byteLength(value.slice(0, middle)) <= maxBytes) low = middle; + else high = middle - 1; + } + return value.slice(0, low); +} + +function createGoalLogSink(logPath: string): (output: SupervisedDockerOutput) => Promise { + let usedBytes: number | undefined; + return async output => { + usedBytes ??= await stat(logPath).then(value => value.size).catch(() => 0); + const remaining = MAX_GOAL_LOG_BYTES - usedBytes; + if (remaining <= 0) return; + const outputData = output.data; + // This is a second, independent allowlist at the persistence boundary. + // Never serialize the delivered object wholesale: callers and adapters + // can supply structurally valid objects with secret-bearing excess keys. + const base = { + goalId: output.goalId, + sessionId: output.sessionId, + controllerEpoch: output.controllerEpoch, + turnId: output.turnId, + executionId: output.executionId, + attemptId: output.attemptId, + worktreeFingerprint: output.worktreeFingerprint, + sequence: output.sequence, + recordedAt: output.recordedAt, + channel: output.channel, + }; + const overhead = Buffer.byteLength(`${JSON.stringify({ ...base, data: '', truncated: true })}\n`); + const data = utf8Prefix(outputData, Math.max(0, remaining - overhead)); + const line = `${JSON.stringify({ ...base, data, truncated: data !== outputData })}\n`; + const lineBytes = Buffer.byteLength(line); + if (lineBytes > remaining) return; + await appendFile(logPath, line, { encoding: 'utf8', mode: 0o600 }); + usedBytes += lineBytes; + }; +} + +async function validateCredentialMounts( + mounts: ReadonlyArray, + providerHomeTarget: string, + allowedMounts: ReadonlySet, +): Promise { + const home = path.posix.normalize(providerHomeTarget).replace(/\/+$/, ''); + for (const mount of mounts) { + const source = await canonicalCredentialSource(mount.source); + const target = canonicalCredentialTarget(mount.target); + const provider = mount.provider ?? credentialProviderForTarget(target) ?? credentialProviderForHome(home); + if (!provider || !isProviderCredentialTarget(provider, target)) { + throw new Error('Credential mount target is not owned by Claude, Codex, or Antigravity'); + } + if (!allowedMounts.has(`${provider}\0${source}\0${target}`) + && !allowedMounts.has(`${source}\0${target}`)) { + throw new Error('Credential mount source and target pair is not explicitly allow-listed'); + } + if (target === home) throw new Error('A credential file may not replace the writable provider home directory'); + if (target === '/workspace' || target.startsWith('/workspace/')) { + throw new Error('Credentials may not be mounted inside the writable workspace'); + } + } +} + +function credentialProviderForTarget(target: string): GoalCredentialMount['provider'] { + if (target === '/home/node/.claude.json' || target.startsWith('/home/node/.claude/')) return 'claude'; + if (target.startsWith('/home/node/.codex/')) return 'codex'; + if (target.startsWith('/home/node/.gemini/')) return 'antigravity'; + return undefined; +} + +function credentialProviderForHome(home: string): GoalCredentialMount['provider'] { + if (home === '/home/node/.claude') return 'claude'; + if (home === '/home/node/.codex') return 'codex'; + if (home === '/home/node/.gemini') return 'antigravity'; + return undefined; +} + +function isProviderCredentialTarget(provider: NonNullable, target: string): boolean { + return credentialProviderForTarget(target) === provider || target === '/home/node/.creds'; +} + +function assertContainerOperationFence( + request: StartGoalContainerRequest | StartGoalOpenContainerRequest, + fence: import('./contract.js').GoalProviderOperationFence, + scope: 'turn' | 'open', +): void { + const turnId = scope === 'turn' ? (request as StartGoalContainerRequest).turnId : undefined; + if (fence.goalId !== request.goalId || fence.sessionId !== request.sessionId + || fence.controllerEpoch !== request.controllerEpoch + || fence.executionId !== request.executionId || fence.attemptId !== request.attemptId + || fence.turnId !== turnId || fence.kind !== scope) { + throw new GoalSessionContractError( + 'Container operation fence does not match its durable execution claim', 'STALE_FENCE', + ); + } +} + +function rebuildStartedContainer(value: SupervisedDockerExecution): SupervisedDockerExecution { + if (!value || typeof value !== 'object' || Object.getPrototypeOf(value) !== Object.prototype + || Object.getOwnPropertySymbols(value).length > 0) { + throw new GoalSessionContractError('Docker returned an invalid supervised execution', 'INVALID_PROVIDER_RESULT'); + } + const descriptors = Object.getOwnPropertyDescriptors(value); + const keys = ['containerName', 'writeInput', 'closeInput', 'cancel', 'completion']; + if (Object.keys(descriptors).length !== keys.length + || keys.some(key => !descriptors[key]?.enumerable || !('value' in descriptors[key]))) { + throw new GoalSessionContractError('Docker returned an invalid supervised execution', 'INVALID_PROVIDER_RESULT'); + } + const name = descriptors.containerName.value as unknown; + if (name !== null && (typeof name !== 'string' || name.length > 128) + || typeof descriptors.writeInput.value !== 'function' + || typeof descriptors.closeInput.value !== 'function' + || typeof descriptors.cancel.value !== 'function' + || !(descriptors.completion.value instanceof Promise) + || Object.getPrototypeOf(descriptors.completion.value) !== Promise.prototype) { + throw new GoalSessionContractError('Docker returned an invalid supervised execution', 'INVALID_PROVIDER_RESULT'); + } + return value; +} + +/** + * Owns goal-scoped container resources and converts duplex byte output into + * normalized, atomically fenced durable events. Provider adapters retain + * responsibility for interpreting structured protocol lines. + */ +export class GoalContainerSupervisor { + private readonly isolation: GoalContainerIsolationPolicy; + private readonly providerFirstEffects?: GoalProviderFirstEffectPort; + private readonly pendingOpen: PendingOpenOwnership; + + constructor( + private readonly baseDirectory: string, + private readonly events: GoalSessionEventSink, + private readonly retention: GoalContainerRetentionPolicy = DEFAULT_GOAL_CONTAINER_RETENTION, + options: GoalContainerSupervisorOptions | GoalContainerIsolationPolicy = {}, + ) { + const resolved = resolveSupervisorOptions(options); + this.isolation = resolved.isolation ?? { + environmentKeys: [], worktreePaths: [], providerHomeTargets: [], credentialMounts: [], + }; + this.providerFirstEffects = resolved.providerFirstEffects; + this.pendingOpen = new PendingOpenOwnership(resolved.dockerPath); + validateAbsolutePath(baseDirectory, 'Goal container base directory'); + } + + async start(request: StartGoalContainerRequest): Promise<{ layout: GoalContainerLayout; execution: SupervisedDockerExecution }> { + return this.startScoped(request, 'turn'); + } + + async startOpen(request: StartGoalOpenContainerRequest): Promise<{ layout: GoalContainerLayout; execution: SupervisedDockerExecution }> { + return this.startScoped(request, 'open'); + } + + /** Idempotently releases a process still owned by its exact eager-open attempt. */ + async cancelPendingOpen(claim: Readonly): Promise { await this.pendingOpen.cancel(claim); } + + async cancelPendingOpenAttempt(identity: { + goalId: string; sessionId: string; attemptId: string; deterministicOpenKey?: string; + }): Promise { + await this.pendingOpen.cancelIdentity(identity); } + + /** Transfers cleanup ownership to the now-persisted provider session. */ + transferPendingOpen(claim: Readonly): void { + this.pendingOpen.transfer(claim); + } + + private async startScoped( + request: StartGoalContainerRequest | StartGoalOpenContainerRequest, + scope: 'turn' | 'open', + ): Promise<{ layout: GoalContainerLayout; execution: SupervisedDockerExecution }> { + assertSafeCallerTurnIdentity({ + turnId: scope === 'turn' ? (request as StartGoalContainerRequest).turnId : 'open', + executionId: request.executionId, + attemptId: request.attemptId, + }); + const worktreePath = await resolveApprovedSource( + request.worktreePath, + new Set(this.isolation.worktreePaths.map(value => path.resolve(value))), + 'Goal worktree path', + ); + if (!(await stat(worktreePath)).isDirectory()) throw new Error('Goal worktree path must be a directory'); + validateProviderHomeTarget( + request.providerHomeTarget, + new Set(this.isolation.providerHomeTargets.map(value => path.posix.normalize(value).replace(/\/+$/, ''))), + ); + if (!request.image.trim()) throw new Error('Goal container image must be non-empty'); + if (!request.worktreeFingerprint.trim()) throw new Error('Goal worktree fingerprint must be non-empty'); + const environment = request.environment ?? {}; + validateEnvironment(environment, new Set(this.isolation.environmentKeys)); + const credentialMounts = request.credentialMounts ?? []; + await validateCredentialMounts( + credentialMounts, + request.providerHomeTarget, + new Set((this.isolation.credentialMounts ?? []).map(mount => + `${mount.provider ?? credentialProviderForTarget(path.posix.normalize(mount.target).replace(/\/+$/, '')) ?? credentialProviderForHome(path.posix.normalize(request.providerHomeTarget).replace(/\/+$/, '')) ?? ''}\0${path.resolve(mount.source)}\0${path.posix.normalize(mount.target).replace(/\/+$/, '')}`)), + ); + const layout = scope === 'turn' + ? buildGoalContainerLayout(this.baseDirectory, request as StartGoalContainerRequest) + : buildGoalOpenContainerLayout(this.baseDirectory, request as StartGoalOpenContainerRequest); + await Promise.all([ + mkdir(layout.providerHome, { recursive: true, mode: 0o700 }), + mkdir(path.dirname(layout.logPath), { recursive: true, mode: 0o700 }), + ]); + const appendGoalLog = createGoalLogSink(layout.logPath); + let observerSubscribed = request.outputObserver !== undefined; + // Explicit public DTOs: the start request also carries commands, + // environment values, mounts, host paths, task IDs, and arbitrary excess + // properties. None of those may cross the durable event boundary. + const eventFence = { + goalId: request.goalId, + sessionId: request.sessionId, + controllerEpoch: request.controllerEpoch, + ...(scope === 'turn' ? { turnId: (request as StartGoalContainerRequest).turnId } : {}), + }; + const eventExecution: GoalExecutionIdentity = { + executionId: request.executionId, + attemptId: request.attemptId, + }; + const operationFence = request.operationFence; + assertContainerOperationFence(request, operationFence, scope); + if (!this.providerFirstEffects) throw new GoalSessionContractError( + 'Container start requires the authoritative provider first-effect gate', 'FIRST_EFFECT_GATE_MISSING', + ); + + const dockerArgs = [ + 'run', '--rm', '--name', layout.containerName, + '--mount', `type=bind,src=${layout.providerHome},dst=${request.providerHomeTarget}`, + '--mount', `type=bind,src=${worktreePath},dst=/workspace`, + ...credentialMounts.flatMap(mount => ['--mount', `type=bind,src=${mount.source},dst=${mount.target},readonly`]), + '--workdir', '/workspace', + // Names only: values are forwarded through the docker client's own + // environment so secrets never appear in argv/process listings. + ...Object.keys(environment).flatMap(name => ['--env', name]), + request.image, + ...request.command, + ]; + const execution = await this.providerFirstEffects.start( + operationFence, 'container_spawn', () => { + const started = executeSupervisedDockerCommand(dockerArgs, { + goalId: request.goalId, + sessionId: request.sessionId, + controllerEpoch: request.controllerEpoch, + turnId: scope === 'turn' ? (request as StartGoalContainerRequest).turnId : undefined, + scope, + openKey: scope === 'open' + ? (request as StartGoalOpenContainerRequest).deterministicOpenKey : undefined, + executionId: request.executionId, + attemptId: request.attemptId, + worktreeFingerprint: request.worktreeFingerprint, + taskId: request.taskId, + operationGeneration: operationFence.generation, + operationKind: operationFence.kind, + operationId: operationFence.operationId, + operationLeaseExpiresAt: operationFence.leaseExpiresAt, + signal: request.signal, + timeout: request.timeout, + env: environment, + durableOutput: async output => { + const safeOutput = sanitizeGoalSessionEvent({ + type: 'output', + channel: output.channel, + data: output.data, + }); + if (safeOutput.type !== 'output') throw new Error('Output sanitizer returned an invalid event'); + const result = scope === 'turn' + ? await this.events.append(eventFence as GoalSessionFence, eventExecution, safeOutput) + : await this.events.appendControl(eventFence, eventExecution, safeOutput); + if (!result.accepted) { + throw new StaleGoalSessionFenceError(`Container output rejected by durable sink: ${result.reason}`); + } + await appendGoalLog({ ...output, channel: safeOutput.channel, data: safeOutput.data }); + if (observerSubscribed && request.outputObserver) { + let disposition: void | 'unsubscribe'; + try { + disposition = await request.outputObserver.next(Object.freeze({ + goalId: output.goalId, sessionId: output.sessionId, + controllerEpoch: output.controllerEpoch, turnId: output.turnId, + executionId: output.executionId, attemptId: output.attemptId, + worktreeFingerprint: output.worktreeFingerprint, sequence: output.sequence, + operationGeneration: output.operationGeneration, + operationKind: output.operationKind, operationId: output.operationId, + operationLeaseExpiresAt: output.operationLeaseExpiresAt, + recordedAt: output.recordedAt, channel: output.channel, data: output.data, + })); + } catch { + throw new GoalSessionContractError( + 'Provider output consumer failed safely', 'PROVIDER_OPERATION_FAILED', + ); + } + if (disposition === 'unsubscribe') observerSubscribed = false; + } + }, + }); + if (scope === 'open') { + this.pendingOpen.register({ + executionId: request.executionId, attemptId: request.attemptId, + deterministicOpenKey: (request as StartGoalOpenContainerRequest).deterministicOpenKey, + operationGeneration: operationFence.generation, operationFence, + }, started); + } + return startedProviderEffect(Promise.resolve(started), () => + started.cancel(new Error('Authoritative Docker-start transaction failed'))); + }, + rebuildStartedContainer, + ); + // Completion notifications are observed and rebuilt; they never create + // an unhandled rejection or expose a subprocess exception to an adapter. + void execution.completion.then(async () => { + if (observerSubscribed) await request.outputObserver?.complete?.(); + }, async () => { + if (observerSubscribed) await request.outputObserver?.error?.( + new GoalSessionContractError('Supervised provider output failed safely', 'PROVIDER_OPERATION_FAILED'), + ); + }).catch(() => undefined); + return { layout, execution }; + } + + retentionDeadline( + terminalAt: Date, + outcome: 'succeeded' | 'cancelled' | 'failed', + ): Date { + const duration = outcome === 'succeeded' + ? this.retention.succeededMs + : outcome === 'cancelled' + ? this.retention.cancelledMs + : this.retention.failedMs; + return new Date(terminalAt.getTime() + duration); + } + + /** + * Removes only a previously derived, goal-scoped session directory after its + * retention deadline. The target must be a real directory whose lexical path + * is exactly its symlink-resolved path: any symlink is rejected, including an + * in-tree one pointing at a sibling goal's real directory, so cleanup can + * never delete another goal's resources through a redirected session root. + */ + async cleanTerminalSession( + layout: GoalContainerLayout, + terminalAt: Date, + outcome: 'succeeded' | 'cancelled' | 'failed', + currentTime = new Date(), + ): Promise { + return cleanTerminalGoalSession({ + baseDirectory: this.baseDirectory, retention: this.retention, layout, terminalAt, outcome, currentTime, + }); + } +} diff --git a/packages/core/src/agents/goalSession/GoalImmediateModelControls.ts b/packages/core/src/agents/goalSession/GoalImmediateModelControls.ts new file mode 100644 index 000000000..855740d6d --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalImmediateModelControls.ts @@ -0,0 +1,418 @@ +import type { GoalModelChangeAcknowledgement, GoalModelChangeIntent, GoalModelChangeRequest, GoalSessionControlFence, GoalSessionEvent, GoalSessionState } from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { GoalTurnRunner } from './GoalTurnRunner.js'; +import { compactImmediateModelIntents, assertModelControllable, hasUnresolvedImmediateModelIntent, immediateModelIntents, isLiveModelLease, latestImmediateModelIntent, nextModelGeneration, obsoleteModelIntents, replaceImmediateModelIntent, requestedImmediateModelIntent, validateImmediateModelAcknowledgement } from './modelChangeProtocol.js'; +import { resolveModelChangeHistory } from './modelChangeHistory.js'; +import { nextState, persistedSnapshot } from './support.js'; +import { assertSafeProviderIdentifier } from './securityBoundary.js'; +import { rebuildModelAcknowledgement } from './providerResultBoundary.js'; +import { claimModelApplicationIntent } from './modelApplicationLease.js'; +import { compositeOperationId } from './controlOperationIdentity.js'; + +/** Durable generation and convergence protocol for provider model side effects. */ +export abstract class GoalImmediateModelControls extends GoalTurnRunner { + async requestModelChange(request: GoalModelChangeRequest): Promise { + assertSafeProviderIdentifier(request.model); + let state = await this.requireControlledState(request); + if (state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed') { + throw new GoalSessionContractError(`Cannot change model while the session is ${state.status}`, 'SESSION_NOT_CONTROLLABLE'); + } + const sameModelIntent = immediateModelIntents(state).findLast(intent => intent.model === request.model); + const operationId = request.operationId ?? sameModelIntent?.modelChangeId ?? this.controlOperationId('model', state); + // Validate before the exact-history claim so a rejected identity cannot + // allocate order or leave any durable/provider trace. + assertSafeProviderIdentifier(operationId); + const retainedIntent = immediateModelIntents(state).some(intent => intent.modelChangeId === operationId); + const appliesAt = this.adapter.capabilities.modelChange === 'next_turn' ? 'next_turn' : 'next_safe_boundary'; + const historical = await resolveModelChangeHistory( + this.ports.modelChanges, request, { operationId, appliesAt, retainedIntent }, + ); + if (historical) return historical; + const exactRequest = { ...request, operationId }; + if (this.adapter.capabilities.modelChange === 'next_turn') { + const acknowledgement = { requestedModel: request.model, appliesAt: 'next_turn' as const }; + if (state.pendingModelChange === request.model + && state.modelChangeIntent?.model === request.model + && state.modelChangeIntent.modelChangeId === operationId) { + return acknowledgement; + } + const modelChangeId = operationId; + const generation = nextModelGeneration(state); + const intent: GoalModelChangeIntent = { + modelChangeId, model: request.model, requestedAt: new Date().toISOString(), generation, + previousModel: state.currentModel, phase: 'pending', + }; + const superseded = immediateModelIntents(state).map(previous => + previous.phase === 'pending' ? { + ...previous, + phase: 'superseded' as const, + acknowledgement: previous.acknowledgement ?? { + requestedModel: previous.model, appliesAt: 'next_turn' as const, + }, + } : previous); + const intents = compactImmediateModelIntents([...superseded, intent]); + state = await this.commitControlTransition({ + state, + fence: request, + changes: { + requestedModel: request.model, + pendingModelChange: request.model, + modelChangeIntent: intent, + modelChangeIntents: intents, + modelChangeGeneration: generation, + }, + auditEvents: [{ type: 'model_change_acknowledged', ...acknowledgement }], + transitionId: `model-requested:${modelChangeId}`, + }); + for (const previous of superseded) { + if (previous.phase === 'superseded' && previous.acknowledgement) { + await this.ports.modelChanges.settle(request, previous.modelChangeId, previous.acknowledgement); + } + } + return acknowledgement; + } + const acknowledgement = await this.applyImmediateModelChange(exactRequest, state); + await this.ports.modelChanges.settle(request, operationId, acknowledgement); + return acknowledgement; + } + + /** Resumes a durable next-safe-boundary intent after an ambiguous provider/local outcome. */ + protected async resumeImmediateModelChangeIntent( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise { + const intent = latestImmediateModelIntent(state); + if (this.adapter.capabilities.modelChange !== 'next_safe_boundary' + || !intent || !hasUnresolvedImmediateModelIntent(state)) return state; + await this.applyImmediateModelChange({ ...fence, model: intent.model }, state); + return this.requireControlledState(fence); + } + + private async applyImmediateModelChange( + request: GoalModelChangeRequest, + initial: GoalSessionState, + ): Promise { + let state = initial; + const resolved = requestedImmediateModelIntent(state, request); + let { intent } = resolved; + if (intent && intent.modelChangeId !== latestImmediateModelIntent(state)?.modelChangeId + && (intent.phase === 'committed' || intent.phase === 'superseded')) { + return intent.acknowledgement ?? { + requestedModel: intent.model, appliesAt: 'next_safe_boundary', + }; + } + if (!intent) { + const intents = immediateModelIntents(state); + const generation = nextModelGeneration(state); + intent = { + modelChangeId: request.operationId ?? this.controlOperationId('model', state), + model: request.model, + requestedAt: new Date().toISOString(), + generation, + previousModel: state.currentModel, + phase: 'pending', + }; + const retained = compactImmediateModelIntents([...intents, intent]); + state = await this.compareAndSetExact(state, { + requestedModel: request.model, + modelChangeIntent: intent, + modelChangeIntents: retained, + modelChangeGeneration: generation, + }, 'A newer model intent superseded this request'); + } + const intentId = intent.modelChangeId; + if (intent.phase === 'committed' && intent.acknowledgement) { + return this.convergeCachedAcknowledgement(request, intentId); + } + return this.applyImmediateModelGeneration(request, intentId); + } + + private async applyImmediateModelGeneration( + fence: GoalSessionControlFence, + requestedIntentId: string, + ): Promise { + let state = await this.requireControlledState(fence); + assertModelControllable(state); + let intent = immediateModelIntents(state).find(value => value.modelChangeId === requestedIntentId); + if (!intent) throw new StaleGoalSessionFenceError('The requested model generation was superseded'); + assertSafeProviderIdentifier(intent.model); + assertSafeProviderIdentifier(intent.modelChangeId); + ({ state, intent } = await this.claimModelApplication(fence, state, intent)); + const operationGeneration = state.providerOperationGeneration ?? 0; + await this.publishProviderOperationBarrier(fence, operationGeneration); + await this.requireProviderGeneration(fence, operationGeneration); + const operationFence = this.modelOperationFence(fence, operationGeneration, intent); + const acknowledgement = await this.providerResult(() => this.providerFirstEffect(operationFence, () => { + const completion = this.adapter.requestModelChange({ + goalId: fence.goalId, sessionId: fence.sessionId, controllerEpoch: fence.controllerEpoch, + model: intent.model, + modelChangeId: intent.modelChangeId, + applicationGeneration: intent.generation ?? 0, + operationGeneration, + operationFence, + }, persistedSnapshot(state)); + return this.startedProviderEffect(completion, () => this.rollbackProviderPrimitive(operationFence, state)); + }, rebuildModelAcknowledgement), rebuildModelAcknowledgement); + validateImmediateModelAcknowledgement({ ...fence, model: intent.model }, state, acknowledgement); + return this.finishImmediateModelGeneration(fence, intent, acknowledgement); + } + + private async finishImmediateModelGeneration( + fence: GoalSessionControlFence, + intent: GoalModelChangeIntent, + acknowledgement: GoalModelChangeAcknowledgement, + ): Promise { + let state: GoalSessionState; + try { + state = await this.requireControlledState(fence); + } catch (error) { + if (error instanceof StaleGoalSessionFenceError) { + await this.reapplyLatestModelAtLiveFence(fence); + } + throw error; + } + assertModelControllable(state); + const latest = latestImmediateModelIntent(state); + const durableIntent = immediateModelIntents(state) + .find(value => value.modelChangeId === intent.modelChangeId); + if (!durableIntent || durableIntent.applicationToken !== intent.applicationToken) { + await this.reapplyLatestModel(fence, latest); + throw new StaleGoalSessionFenceError('The model application lease was replaced before acknowledgement'); + } + if (latest?.modelChangeId !== intent.modelChangeId) { + await this.reapplyLatestModel(fence, latest); + await this.markModelGenerationSuperseded(fence, intent.modelChangeId, acknowledgement); + throw new StaleGoalSessionFenceError('A newer model intent superseded this provider acknowledgement'); + } + const committed = { + ...intent, + phase: 'committed' as const, + acknowledgement, + applicationToken: undefined, + applicationControllerEpoch: undefined, + leaseExpiresAt: undefined, + }; + const intents = replaceImmediateModelIntent(state, committed); + const auditEvents: Array> = [{ + type: 'model_change_acknowledged', requestedModel: intent.model, appliesAt: acknowledgement.appliesAt, + }]; + if (acknowledgement.effectiveModel) { + auditEvents.push({ + type: 'model_changed', previousModel: intent.previousModel ?? state.currentModel, + model: acknowledgement.effectiveModel, + }); + } + await this.commitControlTransition({ + state, + fence, + changes: { + currentModel: acknowledgement.effectiveModel ?? state.currentModel, + modelChangeIntents: intents, + modelChangeIntent: intents.at(-1), + }, + auditEvents, + transitionId: `model-applied:${intent.modelChangeId}`, + }); + await this.ports.modelChanges.settle(fence, intent.modelChangeId, acknowledgement); + await this.markObsoleteModelGenerations(fence, intent.modelChangeId, false); + return acknowledgement; + } + + private async reapplyLatestModel( + fence: GoalSessionControlFence, + intent: GoalModelChangeIntent | undefined, + ): Promise { + if (!intent) return; + let target = intent; + for (;;) { + let state = await this.requireControlledState(fence); + assertModelControllable(state); + const durable = immediateModelIntents(state) + .find(value => value.modelChangeId === target.modelChangeId); + if (!durable) return; + assertSafeProviderIdentifier(durable.model); + assertSafeProviderIdentifier(durable.modelChangeId); + ({ state, intent: target } = await this.claimModelApplication(fence, state, durable)); + const operationGeneration = state.providerOperationGeneration ?? 0; + await this.publishProviderOperationBarrier(fence, operationGeneration); + await this.requireProviderGeneration(fence, operationGeneration); + const operationFence = this.modelOperationFence(fence, operationGeneration, target); + const acknowledgement = await this.providerResult(() => this.providerFirstEffect(operationFence, () => { + const completion = this.adapter.requestModelChange({ + goalId: fence.goalId, sessionId: fence.sessionId, controllerEpoch: fence.controllerEpoch, + model: target.model, + modelChangeId: target.modelChangeId, + applicationGeneration: target.generation ?? 0, + operationGeneration, + operationFence, + }, persistedSnapshot(state)); + return this.startedProviderEffect(completion, () => this.rollbackProviderPrimitive(operationFence, state)); + }, rebuildModelAcknowledgement), rebuildModelAcknowledgement); + validateImmediateModelAcknowledgement({ ...fence, model: target.model }, state, acknowledgement); + state = await this.requireControlledState(fence); + assertModelControllable(state); + const latest = latestImmediateModelIntent(state); + if (latest?.modelChangeId !== target.modelChangeId + || latest.applicationToken !== target.applicationToken) { + target = latest!; + continue; + } + if (target.phase !== 'committed') { + await this.finishImmediateModelGeneration(fence, target, acknowledgement); + } else { + await this.clearModelApplicationLease(fence, target.modelChangeId, target.applicationToken); + } + return; + } + } + + /** Repairs a provider side effect that completed after controller takeover. */ + private async reapplyLatestModelAtLiveFence(identity: GoalSessionControlFence): Promise { + const state = await this.requireState(identity), intent = latestImmediateModelIntent(state); + if (!intent || state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed') return; + const fence = { goalId: state.goalId, sessionId: state.sessionId, controllerEpoch: state.controllerEpoch }; + await this.reapplyLatestModel(fence, intent); + } + + private async convergeCachedAcknowledgement( + fence: GoalSessionControlFence, + requestedIntentId: string, + ): Promise { + for (;;) { + const state = await this.requireControlledState(fence); + const latest = latestImmediateModelIntent(state); + if (!latest || latest.modelChangeId !== requestedIntentId || !latest.acknowledgement) { + throw new StaleGoalSessionFenceError('A newer model intent superseded the cached acknowledgement'); + } + const blockers = immediateModelIntents(state).filter(intent => + intent.modelChangeId !== latest.modelChangeId + && intent.phase !== 'superseded' + && intent.phase !== 'committed'); + const liveBlocker = blockers.find(intent => isLiveModelLease(intent, state.controllerEpoch)); + if (liveBlocker) { + await new Promise(resolve => setImmediate(resolve)); + continue; + } + await this.reapplyLatestModel(fence, latest); + await this.markObsoleteModelGenerations(fence, latest.modelChangeId, true); + const converged = await this.requireControlledState(fence); + const durable = latestImmediateModelIntent(converged); + if (durable?.modelChangeId === requestedIntentId && durable.acknowledgement + && !immediateModelIntents(converged).some(intent => + intent.modelChangeId !== requestedIntentId + && intent.phase !== 'committed' + && intent.phase !== 'superseded')) return durable.acknowledgement; + } + } + + private async claimModelApplication( + fence: GoalSessionControlFence, + initial: GoalSessionState, + requested: GoalModelChangeIntent, + ): Promise<{ state: GoalSessionState; intent: GoalModelChangeIntent }> { + let state = initial; + for (;;) { + const current = immediateModelIntents(state) + .find(value => value.modelChangeId === requested.modelChangeId); + if (!current) throw new StaleGoalSessionFenceError('The model application generation disappeared'); + if (isLiveModelLease(current, state.controllerEpoch)) { + await new Promise(resolve => setImmediate(resolve)); + state = await this.requireControlledState(fence); + assertModelControllable(state); + continue; + } + const claimed = claimModelApplicationIntent(current, state); + const intents = replaceImmediateModelIntent(state, claimed); + try { + const saved = await this.compareAndSetExact(state, { + modelChangeIntents: intents, + modelChangeIntent: intents.at(-1), + }, 'A newer model operation superseded the provider-call lease'); + return { state: saved, intent: claimed }; + } catch (error) { + if (!(error instanceof StaleGoalSessionFenceError)) throw error; + state = await this.requireControlledState(fence); + } + } + } + + private async clearModelApplicationLease( + fence: GoalSessionControlFence, + modelChangeId: string, + applicationToken: string | undefined, + ): Promise { + for (;;) { + const state = await this.requireControlledState(fence); + const intent = immediateModelIntents(state).find(value => value.modelChangeId === modelChangeId); + if (!intent || (applicationToken && intent.applicationToken !== applicationToken)) return; + if (!intent.applicationToken) return; + const cleared = { + ...intent, + applicationToken: undefined, + applicationControllerEpoch: undefined, + leaseExpiresAt: undefined, + }; + const intents = replaceImmediateModelIntent(state, cleared); + const saved = await this.ports.state.compareAndSet(state, nextState(state, { + modelChangeIntents: intents, + modelChangeIntent: intents.at(-1), + })); + if (saved) return; + } + } + + private async markModelGenerationSuperseded( + fence: GoalSessionControlFence, + modelChangeId: string, + acknowledgement: GoalModelChangeAcknowledgement, + ): Promise { + const state = await this.requireControlledState(fence); + const intent = immediateModelIntents(state).find(value => value.modelChangeId === modelChangeId); + if (!intent || intent.phase === 'superseded') return; + const intents = replaceImmediateModelIntent(state, { + ...intent, phase: 'superseded', acknowledgement, + applicationToken: undefined, applicationControllerEpoch: undefined, leaseExpiresAt: undefined, + }); + await this.commitControlTransition({ + state, fence, + changes: { + modelChangeIntents: intents, modelChangeIntent: intents.at(-1), + }, + auditEvents: [{ + type: 'model_change_acknowledged', requestedModel: intent.model, + appliesAt: acknowledgement.appliesAt, + }], + transitionId: `model-superseded:${modelChangeId}`, + }); + await this.ports.modelChanges.settle(fence, modelChangeId, acknowledgement); + } + + private async markObsoleteModelGenerations( + fence: GoalSessionControlFence, + latestModelChangeId: string, + reconciled: boolean, + ): Promise { + const state = await this.requireControlledState(fence); + const { changed, intents } = obsoleteModelIntents(state, latestModelChangeId, reconciled); + if (!changed) return; + await this.compareAndSetExact(state, { + modelChangeIntents: intents, + modelChangeIntent: intents.at(-1), + }, 'A newer operation superseded obsolete model recovery'); + } + + private modelOperationFence( + fence: GoalSessionControlFence, + generation: number, + intent: GoalModelChangeIntent, + ) { + return this.providerOperationFence( + fence, generation, { + kind: 'model', operationId: compositeOperationId( + 'model', intent.modelChangeId, intent.applicationToken ?? 'unclaimed', + ), + leaseExpiresAt: intent.leaseExpiresAt, + }, + ); + } +} diff --git a/packages/core/src/agents/goalSession/GoalSessionControls.ts b/packages/core/src/agents/goalSession/GoalSessionControls.ts new file mode 100644 index 000000000..6efa0456e --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalSessionControls.ts @@ -0,0 +1,263 @@ +import type { + GoalExecutionIdentity, + GoalMessageDeliveryOutcome, + GoalPauseAcknowledgement, + GoalPauseRequest, + GoalSessionControlFence, + GoalSessionState, + GoalSteeringCommand, +} from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { GoalCancellationControls } from './GoalCancellationControls.js'; +import { assertCredentialFreeRecoveryMetadata, sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +import { safeDiagnostic, safeFailureDiagnostic, sanitizeGoalSessionEvent } from './securityBoundary.js'; +import { + assertProviderIdentity, + controlExecutionIdentity, + persistedSnapshot, +} from './support.js'; +import { + rebuildMessageAcknowledgement, rebuildPauseAcknowledgement, rebuildProviderSnapshot, +} from './providerResultBoundary.js'; +import { assertSafeCallerSteeringIdentity } from './safeIdentifier.js'; + +/** Capability-aware steering, pause, resume, model, and cancellation controls. */ +export abstract class GoalSessionControls extends GoalCancellationControls { + async deliverMessage(request: GoalSteeringCommand): Promise { + assertSafeCallerSteeringIdentity(request); + let state = await this.requireActiveTurnState(request); + const execution = this.activeExecution(state); + if ((request.executionId && request.executionId !== execution.executionId) + || (request.attemptId && request.attemptId !== execution.attemptId)) { + throw new StaleGoalSessionFenceError('Steering request does not own the exact current provider attempt'); + } + const pending = (await this.ports.messages.listPending(request)).sort((a, b) => a.sequence - b.sequence); + const message = pending.find(value => value.messageId === request.messageId); + if (!message) { + return { outcome: 'acknowledged', messageId: request.messageId, acknowledgement: 'already_acknowledged' }; + } + if (this.adapter.capabilities.steering === 'next_turn') { + return { outcome: 'unsupported_same_turn', messageId: request.messageId, supportedBoundary: 'next_turn' }; + } + if (pending[0]?.messageId !== request.messageId) { + throw new GoalSessionContractError( + `Corrective message "${request.messageId}" is out of order; "${pending[0]?.messageId}" must be delivered first`, + 'MESSAGE_OUT_OF_ORDER', + ); + } + if (!this.adapter.deliverMessage) { + throw new GoalSessionContractError('Provider declares active-turn steering without implementing it', 'CAPABILITY_METHOD_MISSING'); + } + sanitizeGoalSessionEvent({ type: 'message_acknowledged', messageId: request.messageId }); + state = await this.requireActiveAttemptState(request, execution); + const operationGeneration = state.providerOperationGeneration ?? 0; + await this.publishProviderOperationBarrier(request, operationGeneration); + await this.requireTurnProviderGeneration(request, execution, operationGeneration); + const operationFence = this.providerOperationFence( + request, operationGeneration, { + kind: 'steer', operationId: request.messageId, turnId: request.turnId, + executionId: execution.executionId, attemptId: execution.attemptId, + }, + ); + const acknowledgement = await this.providerResult(() => this.providerFirstEffect(operationFence, () => { + const completion = this.adapter.deliverMessage!({ + goalId: request.goalId, sessionId: request.sessionId, + controllerEpoch: request.controllerEpoch, turnId: request.turnId, + ...execution, operationGeneration, operationFence, + messageId: request.messageId, body: safeDiagnostic(message.body, '[redacted corrective message]'), + }, persistedSnapshot(state)); + return this.startedProviderEffect(completion, () => this.rollbackProviderPrimitive(operationFence, state)); + }, rebuildMessageAcknowledgement), rebuildMessageAcknowledgement); + if (acknowledgement.messageId !== request.messageId) { + throw new GoalSessionContractError('Provider acknowledged a different corrective message', 'MESSAGE_ACK_MISMATCH'); + } + const stillOwned = await this.requireActiveTurnState(request); + if (stillOwned.version !== state.version + || stillOwned.activeTurn?.attemptId !== state.activeTurn?.attemptId) { + throw new StaleGoalSessionFenceError('A newer operation superseded message delivery'); + } + const result = await this.ports.messages.acknowledgeWithEvent(request, execution, request.messageId); + if (result === 'stale_fence') throw new StaleGoalSessionFenceError(); + if (result === 'not_found') { + throw new GoalSessionContractError('Corrective message disappeared before acknowledgement', 'MESSAGE_NOT_FOUND'); + } + return { outcome: 'acknowledged', messageId: request.messageId, acknowledgement: result }; + } + + async requestPause(request: GoalPauseRequest): Promise { + if (this.adapter.capabilities.pause === 'after_turn') return this.requestAfterTurnPause(request); + let state = await this.requireControlledState(request); + if (state.status !== 'running' && state.status !== 'pause_requested') { + throw new GoalSessionContractError(`Cannot pause a session while it is ${state.status}`, 'SESSION_NOT_RUNNING'); + } + if (state.status === 'running') { + const activeTurn = state.activeTurn; + if (!activeTurn) throw new StaleGoalSessionFenceError('No active turn owns the pause request'); + state = await this.commitControlTransition({ + state, + fence: request, + changes: { + status: 'pause_requested', + activeTurn: { ...activeTurn, status: 'pause_requested' }, + resumeIntent: undefined, + completedResume: undefined, + providerOperationGeneration: (state.providerOperationGeneration ?? 0) + 1, + }, + auditEvents: [{ type: 'pause_requested', appliesAt: 'next_safe_boundary' }], + transitionId: this.controlOperationId('pause-active-requested', state), + }); + } + if (!this.adapter.requestPause) { + throw new GoalSessionContractError('Provider declares active-turn pause without implementing it', 'CAPABILITY_METHOD_MISSING'); + } + const operationGeneration = state.providerOperationGeneration ?? 0; + await this.publishProviderOperationBarrier(request, operationGeneration); + await this.requireProviderGeneration(request, operationGeneration); + const operationFence = this.providerOperationFence( + request, operationGeneration, + { + kind: 'pause', operationId: this.controlOperationId('pause', state), + turnId: state.activeTurn?.turnId, executionId: state.activeTurn?.executionId, + attemptId: state.activeTurn?.attemptId, + }, + ); + const acknowledgement = await this.providerResult(() => this.providerFirstEffect(operationFence, () => { + const completion = this.adapter.requestPause!({ + goalId: request.goalId, sessionId: request.sessionId, controllerEpoch: request.controllerEpoch, + reason: request.reason ? safeFailureDiagnostic(request.reason, 'Operator requested pause') : undefined, + operationGeneration, operationFence, + }, persistedSnapshot(state)); + return this.startedProviderEffect(completion, () => this.rollbackProviderPrimitive(operationFence, state)); + }, rebuildPauseAcknowledgement), rebuildPauseAcknowledgement); + if (acknowledgement.appliesAt === 'after_turn') { + throw new GoalSessionContractError('Active-turn provider returned an after-turn pause acknowledgement', 'CAPABILITY_ACK_MISMATCH'); + } + const stillOwned = await this.requireControlledState(request); + if (stillOwned.version !== state.version || stillOwned.status === 'terminated' || stillOwned.status === 'failed') { + throw new StaleGoalSessionFenceError('A newer operation superseded the pause acknowledgement'); + } + if (acknowledgement.boundaryReached) { + const activeTurn = state.activeTurn; + if (!activeTurn) throw new StaleGoalSessionFenceError('No active turn owns the pause boundary'); + state = await this.commitControlTransition({ + state, + fence: request, + changes: { + status: 'paused', + activeTurn: { ...activeTurn, status: 'paused' }, + }, + auditEvents: [{ type: 'pause_boundary', ...acknowledgement.boundaryReached }], + transitionId: this.controlOperationId('pause-active-boundary', state), + }); + } + return acknowledgement; + } + + async resumeSession(request: GoalSessionControlFence): Promise { + if (this.adapter.capabilities.pause !== 'after_turn') { + throw new GoalSessionContractError( + 'An active-turn provider resumes through resumeTurn, not a new turn boundary', + 'UNSUPPORTED_AFTER_TURN_RESUME', + ); + } + let state = await this.requireControlledState(request); + if (state.status === 'idle' && state.completedResume?.kind === 'after_turn' + && state.completedResume.controllerEpoch === request.controllerEpoch) return state; + if (state.status !== 'paused' || state.activeTurn) { + throw new GoalSessionContractError(`Cannot resume a session while it is ${state.status}`, 'SESSION_NOT_PAUSED'); + } + const execution = controlExecutionIdentity(state); + state = await this.claimResumeOperation(request, state, { kind: 'after_turn', execution }); + state = await this.promoteResumeOperation(request, state); + const intent = state.resumeIntent!; + state = await this.requireLiveResumeOperation(request, intent.operationId, intent.operationGeneration); + let snapshot; + try { + const providerRequest = this.providerResumeRequest(request, intent); + await this.publishProviderOperationBarrier(request, intent.operationGeneration); + await this.requireProviderGeneration(request, intent.operationGeneration); + snapshot = await this.providerResult(() => this.providerFirstEffect( + providerRequest.operationFence, + () => { + const completion = this.adapter.resumeSession(providerRequest, persistedSnapshot(state)); + return this.startedProviderEffect( + completion, + () => this.rollbackProviderPrimitive(providerRequest.operationFence, state), + ); + }, + value => rebuildProviderSnapshot(value, this.adapter.provider), + ), value => rebuildProviderSnapshot(value, this.adapter.provider)); + } catch (error) { + await this.expireResumeOperation(request, intent.operationId, intent.operationGeneration); + throw error; + } + assertCredentialFreeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider); + assertProviderIdentity(state, snapshot); + state = await this.requireLiveResumeOperation(request, intent.operationId, intent.operationGeneration); + return this.commitControlTransition({ + state, + fence: request, + changes: { + providerSessionId: snapshot.providerSessionId, + recoveryMetadata: sanitizeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider), + currentModel: snapshot.model ?? state.currentModel, + status: 'idle', + resumeIntent: { ...intent, phase: 'settled' }, + completedResume: { + operationId: intent.operationId, operationGeneration: intent.operationGeneration, + kind: intent.kind, controllerEpoch: intent.controllerEpoch, + }, + }, + auditEvents: [{ type: 'session_resumed' }], + transitionId: `resume-settled:${intent.operationId}:${intent.operationGeneration}`, + execution, + }); + } + + private async requestAfterTurnPause(request: GoalPauseRequest): Promise { + let state = await this.requireControlledState(request); + if (state.status === 'paused') return { appliesAt: 'after_turn' }; + if (state.status !== 'idle' && state.status !== 'running' && state.status !== 'pause_requested') { + throw new GoalSessionContractError(`Cannot pause a session while it is ${state.status}`, 'SESSION_NOT_CONTROLLABLE'); + } + if (state.status === 'idle') { + const boundaryReached = { boundary: 'after_turn' }; + state = await this.commitControlTransition({ + state, + fence: request, + changes: { + status: 'paused', resumeIntent: undefined, completedResume: undefined, + providerOperationGeneration: (state.providerOperationGeneration ?? 0) + 1, + }, + auditEvents: [ + { type: 'pause_requested', appliesAt: 'after_turn' }, + { type: 'pause_boundary', ...boundaryReached }, + ], + transitionId: this.controlOperationId('pause-after-turn', state), + }); + await this.publishProviderOperationBarrier(request, state.providerOperationGeneration ?? 0); + return { appliesAt: 'after_turn', boundaryReached }; + } + if (state.status === 'running') { + state = await this.commitControlTransition({ + state, + fence: request, + changes: { + status: 'pause_requested', + activeTurn: state.activeTurn ? { ...state.activeTurn, status: 'pause_requested' } : state.activeTurn, + pendingAfterTurnPause: true, + resumeIntent: undefined, + completedResume: undefined, + }, + auditEvents: [{ type: 'pause_requested', appliesAt: 'after_turn' }], + transitionId: this.controlOperationId('pause-after-turn', state), + }); + } + return { appliesAt: 'after_turn' }; + } + + private activeExecution(state: GoalSessionState): GoalExecutionIdentity { + if (!state.activeTurn) throw new StaleGoalSessionFenceError('No active turn owns this operation'); + return { executionId: state.activeTurn.executionId, attemptId: state.activeTurn.attemptId }; + } +} diff --git a/packages/core/src/agents/goalSession/GoalSessionCore.ts b/packages/core/src/agents/goalSession/GoalSessionCore.ts new file mode 100644 index 000000000..277ca22fd --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalSessionCore.ts @@ -0,0 +1,466 @@ +import { randomUUID } from 'node:crypto'; +import type { + GoalExecutionIdentity, + GoalSessionAdapter, + GoalSessionControlFence, + GoalSessionEvent, + GoalSessionFence, + GoalSessionIdentity, + GoalSessionRuntimePorts, + GoalSessionState, + GoalTerminalCommit, + GoalResumeKind, + GoalResumeIntent, + GoalProviderResumeRequest, GoalProviderOperationFence, GoalProviderEffectStage, GoalStartedProviderEffect, +} from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { assertSafeProviderIdentifier, safeProviderException, sanitizeGoalSessionEvent } from './securityBoundary.js'; +import { decodeDurableGoalSessionState } from './durableStateSecurity.js'; +import { boundedProviderBoundary, expireResumeLease } from './providerBarrierProtocol.js'; +import { rebuildIteratorResult, untrustedProviderResult } from './providerResultBoundary.js'; +import { + controlExecutionIdentity, + nextState, + validateControlFence, +} from './support.js'; +import { completesAtAfterTurnPause, needsAfterTurnPauseAudit } from './turnCompletionProtocol.js'; +import { compositeOperationId, controlOperationId, mintFreshAttemptId } from './controlOperationIdentity.js'; +import { + createProviderOperationFence, createProviderResumeRequest, providerFirstEffectStream, + rollbackStartedProviderPrimitive, startedProviderEffect, +} from './providerEffectProtocol.js'; +import { assertGoalProviderEffectStage } from './providerOperationBoundary.js'; + +/** + * Low-level, fenced state and event primitives shared by every high-level goal + * session operation. It deliberately separates two fencing scopes: + * + * - control scope: goal/session/epoch only, used for pause, resume, model + * change, cancel and reconciliation, and for the audit events they emit even + * when no turn is active; and + * - turn scope: control scope plus the exact active turn, used for turn output. + */ +export abstract class GoalSessionCore { + constructor( + protected readonly adapter: GoalSessionAdapter, + protected readonly ports: GoalSessionRuntimePorts, + private readonly createAttemptId: () => string = randomUUID, + ) {} + + protected mintAttemptId(): string { + return this.createAttemptId(); + } + + protected mintFreshAttemptId(previousAttemptId: string): string { + return mintFreshAttemptId(previousAttemptId, () => this.mintAttemptId()); + } + + /** Stable, non-secret identity for a control operation claimed at one state version. */ + protected controlOperationId(kind: string, state: GoalSessionState): string { + return controlOperationId(kind, state); + } + + protected async claimResumeOperation( + fence: GoalSessionControlFence, + state: GoalSessionState, + options: { kind: GoalResumeKind; execution: GoalExecutionIdentity; turnId?: string }, + ): Promise { + const { kind, execution, turnId } = options; + const previous = state.resumeIntent; + if (previous && previous.phase !== 'settled' + && Date.parse(previous.leaseExpiresAt) > Date.now()) { + throw new GoalSessionContractError('Another process owns the durable resume lease', 'RESUME_IN_PROGRESS'); + } + const generation = (state.providerOperationGeneration ?? 0) + 1; + const intent: GoalResumeIntent = { + ...execution, + operationId: previous?.kind === kind ? previous.operationId : this.controlOperationId(`resume-${kind}`, state), + operationGeneration: generation, + kind, controllerEpoch: fence.controllerEpoch, turnId, + claimedAt: new Date().toISOString(), + leaseExpiresAt: new Date(Date.now() + 30_000).toISOString(), + phase: 'claimed', + }; + return this.compareAndSetExact(state, { + providerOperationGeneration: generation, + resumeIntent: intent, + completedResume: undefined, + }, 'A newer operation claimed the resume lease'); + } + + protected async promoteResumeOperation(fence: GoalSessionControlFence, state: GoalSessionState): Promise { + const intent = state.resumeIntent; + if (!intent || intent.phase !== 'claimed') throw new StaleGoalSessionFenceError('Resume lease is not claimable'); + return this.compareAndSetExact(state, { + resumeIntent: { ...intent, phase: 'provider_in_doubt' }, + }, 'Cancellation or replacement fenced resume before the provider call'); + } + + protected async requireLiveResumeOperation( + fence: GoalSessionControlFence, + operationId: string, + operationGeneration: number, + ): Promise { + const state = await this.requireControlledState(fence); + const intent = state.resumeIntent; + if (!intent || intent.operationId !== operationId + || intent.operationGeneration !== operationGeneration + || state.providerOperationGeneration !== operationGeneration + || intent.phase !== 'provider_in_doubt' + || Date.parse(intent.leaseExpiresAt) <= Date.now() + || state.status !== 'paused') { + throw new StaleGoalSessionFenceError('Resume provider operation was durably preempted or expired'); + } + return state; + } + + protected providerResumeRequest(fence: GoalSessionControlFence, intent: GoalResumeIntent): GoalProviderResumeRequest { + return createProviderResumeRequest(fence, intent); + } + + protected providerOperationFence( + identity: GoalSessionControlFence, + generation: number, + operation: Pick + & Partial>, + ): GoalProviderOperationFence { + return createProviderOperationFence(identity, generation, operation); + } + + protected async publishProviderOperationBarrier( + identity: GoalSessionIdentity, + generation: number, + pendingCancellationId?: string, + ): Promise { + try { + await boundedProviderBoundary(this.adapter.publishOperationBarrier({ + goalId: identity.goalId, + sessionId: identity.sessionId, + generation, + publishedAt: new Date().toISOString(), + pendingCancellationId, + })); + } catch (error) { + throw safeProviderException(error, 'Provider barrier publication failed safely'); + } + } + + protected async providerEffect(effect: () => T | Promise): Promise { + try { return await effect(); } + catch (error) { throw safeProviderException(error); } + } + + /** Starts the primitive while the authoritative state row is transaction-locked. */ + protected providerFirstEffect(fence: GoalProviderOperationFence, effect: () => GoalStartedProviderEffect, + rebuild: (value: T) => R, stage: GoalProviderEffectStage = 'provider_primitive'): Promise { + assertGoalProviderEffectStage(stage); + return this.ports.providerFirstEffects.start(fence, stage, effect, rebuild); + } + + protected startedProviderEffect(completion: Promise, rollbackOrCancel: () => void | Promise): GoalStartedProviderEffect { + return startedProviderEffect(completion, rollbackOrCancel); + } + + protected rollbackProviderPrimitive(fence: GoalProviderOperationFence, state: GoalSessionState): Promise { + return rollbackStartedProviderPrimitive(this.adapter, fence, state); + } + + protected providerFirstEffectStream(fence: GoalProviderOperationFence, + create: () => AsyncIterable): AsyncIterable { + return providerFirstEffectStream( + this.ports.providerFirstEffects, fence, create, + value => rebuildIteratorResult(value) as IteratorResult, + ); + } + + protected async providerResult( + effect: () => T | Promise, + rebuild: (value: Awaited) => R, + ): Promise { + return untrustedProviderResult(effect, rebuild); + } + + protected turnProviderOperationFence( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + generation: number, + ): GoalProviderOperationFence { + return this.providerOperationFence( + fence, generation, + { + kind: 'turn', operationId: compositeOperationId( + 'turn', fence.turnId, execution.executionId, execution.attemptId, + ), + turnId: fence.turnId, executionId: execution.executionId, attemptId: execution.attemptId, + }, + ); + } + + protected async expireResumeOperation( + fence: GoalSessionControlFence, + operationId: string, + operationGeneration: number, + ): Promise { + try { + await expireResumeLease({ + ports: this.ports, fence, operationId, operationGeneration, + load: () => this.requireControlledStateForBarrier(fence), + publish: (state, generation) => this.publishProviderOperationBarrier(state, generation), + }); + } catch (error) { + if (!(error instanceof StaleGoalSessionFenceError)) throw error; + } + } + + protected async requireState(identity: GoalSessionIdentity): Promise { + const state = await this.ports.state.load(identity); + if (!state) throw new GoalSessionContractError('Goal session does not exist', 'SESSION_NOT_FOUND'); + return decodeDurableGoalSessionState(state); + } + + /** Loads state for a session-scoped control operation, rejecting stale epochs. */ + protected async requireControlledState(fence: GoalSessionControlFence): Promise { + const state = await this.requireControlledStateForBarrier(fence); + if (state.providerBarrierIntent?.phase === 'pending') { + throw new StaleGoalSessionFenceError('A durable provider invalidation fenced this operation'); + } + return state; + } + + /** Cancellation/reopen repair is the only path allowed to observe a pending barrier. */ + protected async requireControlledStateForBarrier(fence: GoalSessionControlFence): Promise { + validateControlFence(fence); + const state = await this.requireState(fence); + if (state.controllerEpoch !== fence.controllerEpoch) throw new StaleGoalSessionFenceError(); + return state; + } + + /** Loads state for a turn-scoped operation; the fence must own the active turn. */ + protected async requireActiveTurnState(fence: GoalSessionFence): Promise { + assertSafeProviderIdentifier(fence.turnId); + const state = await this.requireControlledState(fence); + if (!state.activeTurn || state.activeTurn.turnId !== fence.turnId) { + throw new StaleGoalSessionFenceError('Turn fence does not own the active session turn'); + } + if (['completed', 'cancelled', 'failed'].includes(state.activeTurn.status) + || ['cancelling', 'terminated', 'failed'].includes(state.status)) { + throw new StaleGoalSessionFenceError('Turn fence no longer owns a live session turn'); + } + return state; + } + + /** Loads the exact provider invocation, not merely the logical turn. */ + protected async requireActiveAttemptState( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + ): Promise { + const state = await this.requireActiveTurnState(fence); + if (state.activeTurn?.executionId !== execution.executionId + || state.activeTurn.attemptId !== execution.attemptId) { + throw new StaleGoalSessionFenceError('A newer recovery attempt owns this turn'); + } + return state; + } + + protected async requireProviderGeneration( + fence: GoalSessionControlFence, + generation: number, + ): Promise { + const state = await this.requireControlledState(fence); + if ((state.providerOperationGeneration ?? 0) !== generation) { + throw new StaleGoalSessionFenceError('Provider operation generation was durably invalidated'); + } + return state; + } + + protected async requireTurnProviderGeneration( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + generation: number, + ): Promise { + const state = await this.requireActiveAttemptState(fence, execution); + if ((state.providerOperationGeneration ?? 0) !== generation + || (state.activeTurn?.providerOperationGeneration ?? 0) !== generation) { + throw new StaleGoalSessionFenceError('Turn provider operation generation was durably invalidated'); + } + return state; + } + + protected async updateControlledState( + fence: GoalSessionControlFence, + update: (state: GoalSessionState) => Partial, + ): Promise { + return this.compareAndSetLoop(() => this.requireControlledState(fence), update); + } + + protected async updateActiveTurnState( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + update: (state: GoalSessionState) => Partial, + ): Promise { + return this.compareAndSetLoop(() => this.requireActiveAttemptState(fence, execution), update); + } + + /** One-shot CAS for an operation that must not retry over a newer intent. */ + protected async compareAndSetExact( + expected: GoalSessionState, + changes: Partial, + message = 'A newer same-epoch operation superseded this update', + ): Promise { + const saved = await this.ports.state.compareAndSet(expected, nextState(expected, changes)); + if (!saved) throw new StaleGoalSessionFenceError(message); + return saved; + } + + protected async commitTurnCompletion( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + event: Extract, + ): Promise { + const { outcome } = event; + // A pause request can win after the final exact-attempt load but before + // the terminal transaction. Reload and retry under the same attempt so + // the transaction canonically records pause_boundary then completion. + for (let attempt = 0; attempt < 4; attempt += 1) { + const state = await this.requireActiveAttemptState(fence, execution); + const activeTurn = state.activeTurn; + if (!activeTurn) throw new StaleGoalSessionFenceError('Turn fence no longer owns an active turn'); + const existing = state.completedTurns ?? []; + const completedTurns = existing.some(turn => turn.turnId === fence.turnId) + ? existing + : [...existing, { turnId: fence.turnId, ...execution }]; + const afterTurnPaused = completesAtAfterTurnPause(state, outcome, this.adapter.capabilities.pause); + const recordsAfterTurnPause = needsAfterTurnPauseAudit(state, outcome, this.adapter.capabilities.pause); + const next = nextState(state, { + status: outcome === 'cancelled' + ? 'terminated' + : outcome === 'failed' + ? 'failed' + : afterTurnPaused ? 'paused' : 'idle', + failureReason: outcome === 'failed' ? 'Provider reported turn failure safely' : undefined, + activeTurn: afterTurnPaused + ? undefined + : { ...activeTurn, status: outcome === 'succeeded' ? 'completed' : outcome === 'cancelled' ? 'cancelled' : 'failed' }, + completedTurnIds: state.completedTurnIds.includes(fence.turnId) + ? state.completedTurnIds + : [...state.completedTurnIds, fence.turnId], + completedTurns, + pendingAfterTurnPause: undefined, + resumeIntent: undefined, + }); + const completion: GoalTerminalCommit = { + scope: 'turn', + fence, + execution, + auditEvents: recordsAfterTurnPause + ? [{ type: 'pause_boundary', boundary: 'after_turn' }] + : [], + event: sanitizeGoalSessionEvent(event) as Extract, + }; + const saved = await this.ports.terminal.commit(state, next, completion); + if (saved) return saved; + } + throw new StaleGoalSessionFenceError('A newer operation completed or replaced this turn'); + } + + protected async commitControlCompletion( + state: GoalSessionState, + fence: GoalSessionControlFence, + changes: Partial, + event: Extract, + ): Promise { + const execution = controlExecutionIdentity(state); + const saved = await this.ports.terminal.commit(state, nextState(state, changes), { + scope: 'control', fence, execution, auditEvents: [], + event: sanitizeGoalSessionEvent(event) as Extract, + }); + if (!saved) throw new StaleGoalSessionFenceError('A newer operation superseded terminal completion'); + return saved; + } + + /** Commits a nonterminal control state change and its audit events atomically. */ + protected async commitControlTransition( + options: { + state: GoalSessionState; + fence: GoalSessionControlFence; + changes: Partial; + auditEvents: ReadonlyArray>; + transitionId: string; + execution?: GoalExecutionIdentity; + }, + ): Promise { + const { state, fence, changes, auditEvents, transitionId } = options; + const execution = options.execution ?? controlExecutionIdentity(state); + const saved = await this.ports.transitions.commit(state, nextState(state, changes), { + transitionId, + fence, + execution, + auditEvents: auditEvents.map(event => sanitizeGoalSessionEvent(event)) as typeof auditEvents, + }); + if (!saved) throw new StaleGoalSessionFenceError('A newer operation superseded the state/audit transaction'); + return saved; + } + + /** Commits a live-turn state change and provider-stream audit under one exact-attempt fence. */ + protected async commitTurnTransition( + options: { + state: GoalSessionState; + fence: GoalSessionFence; + execution: GoalExecutionIdentity; + update: (state: GoalSessionState) => Partial; + auditEvents: ReadonlyArray>; + transitionId: string; + }, + ): Promise { + const { fence, execution, update, auditEvents, transitionId } = options; + let state = options.state; + for (let attempt = 0; attempt < 4; attempt += 1) { + const saved = await this.ports.transitions.commit(state, nextState(state, update(state)), { + transitionId, + fence, + execution, + auditEvents: auditEvents.map(event => sanitizeGoalSessionEvent(event)) as typeof auditEvents, + turnScoped: true, + }); + if (saved) return saved; + state = await this.requireActiveAttemptState(fence, execution); + } + throw new StaleGoalSessionFenceError('A newer operation repeatedly superseded the turn state/audit transaction'); + } + + private async compareAndSetLoop( + load: () => Promise, + update: (state: GoalSessionState) => Partial, + ): Promise { + for (let attempt = 0; attempt < 4; attempt += 1) { + const state = await load(); + const saved = await this.ports.state.compareAndSet(state, nextState(state, update(state))); + if (saved) return saved; + } + throw new StaleGoalSessionFenceError('Could not persist a fenced session update'); + } + + /** Turn-scoped append; a rejection means this controller no longer owns the turn. */ + protected async append(fence: GoalSessionFence, execution: GoalExecutionIdentity, event: GoalSessionEvent): Promise { + const result = await this.ports.events.append(fence, execution, sanitizeGoalSessionEvent(event)); + if (!result.accepted) throw new StaleGoalSessionFenceError(`Durable event sink rejected output: ${result.reason}`); + } + + /** Turn-scoped append tolerant of losing ownership on an error/cleanup path. */ + protected async appendIfOwned(fence: GoalSessionFence, execution: GoalExecutionIdentity, event: GoalSessionEvent): Promise { + const result = await this.ports.events.append(fence, execution, sanitizeGoalSessionEvent(event)); + if (!result.accepted && result.reason !== 'stale_fence') { + throw new GoalSessionContractError(`Durable event sink rejected output: ${result.reason}`, 'EVENT_REJECTED'); + } + } + + /** Session-scoped control/audit append that does not require an active turn. */ + protected async appendControl( + fence: GoalSessionControlFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + const result = await this.ports.events.appendControl(fence, execution, sanitizeGoalSessionEvent(event)); + if (!result.accepted) throw new StaleGoalSessionFenceError(`Durable control sink rejected event: ${result.reason}`); + } +} diff --git a/packages/core/src/agents/goalSession/GoalSessionRecoveryControls.ts b/packages/core/src/agents/goalSession/GoalSessionRecoveryControls.ts new file mode 100644 index 000000000..81fcb677e --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalSessionRecoveryControls.ts @@ -0,0 +1,425 @@ +import type { + GoalContainerInspection, GoalExecutionIdentity, GoalRepositoryIdentity, GoalRepositoryInspection, + GoalSessionControlFence, GoalSessionIdentity, GoalSessionState, +} from './contract.js'; +import { StaleGoalSessionFenceError } from './errors.js'; +import { GoalSessionControls } from './GoalSessionControls.js'; +import { hasUnresolvedImmediateModelIntent, latestImmediateModelIntent, prepareModelEvidenceForRecoveredAttempt } from './modelChangeProtocol.js'; +import { assertCredentialFreeRecoveryMetadata, sanitizeRecoveryMetadata, scrubDurableRecoveryMetadata } from './recoveryMetadata.js'; +import { + assertLiveRecoveryLease, assertRecoverableExactState, completedRecoveryResult, + isRecoverableStatus, RECOVERY_LEASE_MS, stoppedReconciliationResult, +} from './recoveryOperationProtocol.js'; +import { reconcileRecoveredTurn } from './reconcileRecoveredTurn.js'; +import { sanitizeContainerInspection, sanitizeRepositoryInspection, verifyReconciliationTarget, verifyRecoveredContainer } from './reconciliationIdentity.js'; +import { normalizeRecoveryRepositories } from './repositorySecurity.js'; +import { + assertProviderIdentity, controlExecutionIdentity, nextState, nowIso, + persistedSnapshot, validateEpoch, validateIdentity, +} from './support.js'; +import { fingerprintGoalWorktree } from './worktreeIdentity.js'; +import { safeFailureDiagnostic } from './securityBoundary.js'; +import { expireRecoveryLease } from './providerBarrierProtocol.js'; +import { rebuildReconcileResult } from './providerResultBoundary.js'; +import { RecoveryGuardResult, revalidateRecoveryInspection } from './recoveryRevalidation.js'; + +export type ReconcileGoalSessionResult = { + outcome: 'alive' | 'resumed' | 'failed' | 'blocked'; + reason: string; + state: GoalSessionState; +}; + +type PreparedRecovery = { state: GoalSessionState; fence: GoalSessionControlFence; container: GoalContainerInspection; repository: GoalRepositoryInspection }; + +/** Ownership takeover and cancellation-aware provider recovery operations. */ +export abstract class GoalSessionRecoveryControls extends GoalSessionControls { + async takeover(identity: GoalSessionIdentity, controllerEpoch: number): Promise { + validateIdentity(identity); + validateEpoch(controllerEpoch); + for (let attempt = 0; attempt < 4; attempt += 1) { + let state = await this.requireState(identity); + if (controllerEpoch <= state.controllerEpoch) { + if (controllerEpoch === state.controllerEpoch) return state; + throw new StaleGoalSessionFenceError(); + } + const oldFence = { ...identity, controllerEpoch: state.controllerEpoch }; + if (state.status === 'cancelling') { + return state.cancellationIntent + ? this.resumeClaimedCancellation(oldFence, state) + : this.cancel({ + ...oldFence, + reason: state.failureReason ?? 'Settle cancellation before controller replacement', + }); + } + if (state.status === 'terminated' || state.status === 'failed') { + if (state.providerBarrierIntent?.phase === 'pending') { + return this.repairPendingProviderBarrier(oldFence, state); + } + return state; + } + state = await this.repairPendingProviderBarrier(oldFence, state); + if (state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed') continue; + const generation = (state.providerOperationGeneration ?? 0) + 1; + const operationId = `replacement-e${controllerEpoch}-g${generation}`; + const staged = await this.ports.state.compareAndSet(state, nextState(state, { + // Ownership changes in the same durable claim as invalidation. + // Old-controller appends are fenced even if publication hangs. + controllerEpoch, + providerOperationGeneration: generation, + providerBarrierIntent: { + generation, operationId, kind: 'replacement', phase: 'pending', claimedAt: nowIso(), + }, + })); + if (!staged) continue; + await this.publishProviderOperationBarrier(staged, generation); + const current = await this.requireControlledStateForBarrier({ ...identity, controllerEpoch }); + if (current.providerBarrierIntent?.operationId !== operationId) continue; + const saved = await this.ports.state.compareAndSet(current, nextState(current, { + providerBarrierIntent: { ...current.providerBarrierIntent, phase: 'published' }, + })); + if (saved) return saved; + } + throw new StaleGoalSessionFenceError('Another controller repeatedly changed the session during takeover'); + } + + async reconcile( + identity: GoalSessionIdentity, + controllerEpoch: number, + repository: GoalRepositoryIdentity, + ): Promise { + const committed = await this.committedRecoveryResult(identity, controllerEpoch); + if (committed) return committed; + let prepared: PreparedRecovery | ReconcileGoalSessionResult; + try { + prepared = await this.prepareRecovery(identity, controllerEpoch, repository); + } catch (error) { + if (error instanceof RecoveryGuardResult) return error.result; + throw error; + } + if ('outcome' in prepared) return prepared; + let state: GoalSessionState; + try { + state = await this.revalidatePreparedRecovery(prepared); + } catch (error) { + if (error instanceof RecoveryGuardResult) return error.result; + throw error; + } + const recovery = await this.claimRecoveryAttempt(state, controllerEpoch); + if (!recovery) { + return { + outcome: 'blocked', + reason: 'Another process owns the durable reconciliation lease', + state: await this.requireControlledState(prepared.fence), + }; + } + try { + state = await this.promoteRecoveryAttempt(recovery.state, recovery.execution, controllerEpoch); + } catch (error) { + return this.handleRecoveryPromotionLoss(error, identity, prepared.fence); + } + // This durable reload closes the promotion-to-provider-call gap. A + // cancellation that preempts the token before this await resolves makes + // the call impossible; a cancellation after it resolves treats the call + // as genuinely started and progresses through the provider cancel API. + state = await this.requireLiveRecoveryLease(prepared.fence, recovery.execution, state.recoveryAttempt!.operationToken); + let result: Awaited>; + try { + const operation = state.recoveryAttempt!; + await this.publishProviderOperationBarrier(prepared.fence, operation.operationGeneration); + await this.requireProviderGeneration(prepared.fence, operation.operationGeneration); + const operationFence = this.providerOperationFence( + prepared.fence, operation.operationGeneration, { + kind: 'reconcile', operationId: operation.operationToken, + leaseExpiresAt: operation.leaseExpiresAt, + executionId: recovery.execution.executionId, attemptId: recovery.execution.attemptId, + }, + ); + result = await this.providerResult(() => this.providerFirstEffect(operationFence, () => { + const completion = this.adapter.reconcile({ + goalId: identity.goalId, sessionId: identity.sessionId, ...recovery.execution, controllerEpoch, + operationToken: state.recoveryAttempt!.operationToken, + operationGeneration: state.recoveryAttempt!.operationGeneration, + operationPhase: 'provider_in_doubt', + operationLeaseExpiresAt: state.recoveryAttempt!.leaseExpiresAt, + operationFence, persisted: persistedSnapshot(state), container: prepared.container, + repository: prepared.repository, + }); + return this.startedProviderEffect(completion, () => this.rollbackProviderPrimitive(operationFence, state)); + }, value => rebuildReconcileResult(value, this.adapter.provider)), + value => rebuildReconcileResult(value, this.adapter.provider)); + } catch (error) { + await this.requireLiveRecoveryLease( + prepared.fence, recovery.execution, state.recoveryAttempt!.operationToken, + ); + await this.expireRecoveryLeaseIfOwned(prepared.fence, state.recoveryAttempt!.operationToken); + throw error; + } + state = await this.requireLiveRecoveryLease( + prepared.fence, recovery.execution, state.recoveryAttempt!.operationToken, + ); + return this.persistRecoveryResult(prepared.fence, state, recovery.execution, result); + } + + private async prepareRecovery(identity: GoalSessionIdentity, controllerEpoch: number, repository: GoalRepositoryIdentity): + Promise { + let state = await this.requireState(identity); + if (controllerEpoch < state.controllerEpoch) throw new StaleGoalSessionFenceError(); + if (state.status === 'cancelling') { + const oldFence = { ...identity, controllerEpoch: state.controllerEpoch }; + const cancelled = state.cancellationIntent + ? await this.resumeClaimedCancellation(oldFence, state) + : await this.cancel({ ...oldFence, reason: state.failureReason ?? 'Resume cancellation before recovery takeover' }); + return { + outcome: 'blocked', reason: 'Cancellation completed before reconciliation takeover', state: cancelled, + }; + } + if (controllerEpoch > state.controllerEpoch) state = await this.takeover(identity, controllerEpoch); + const fence = { ...identity, controllerEpoch }; + const guarded = await this.guardReconciliationState(state, fence); + if (guarded) return guarded; + // guardReconciliationState is asynchronous for cancellation recovery. + // Reload once after that gap and synchronously reject every non-live + // state before opening either inspection primitive. + state = await this.requireControlledState(fence); + const stopped = stoppedReconciliationResult(state); + if (stopped) { + if (state.status !== 'cancelling') return stopped; + return (await this.guardReconciliationState(state, fence))!; + } + const repositories = await normalizeRecoveryRepositories(state, repository); + if (!repositories) return this.blockRecovery(fence, state, + 'Recovery repository does not contain a trustworthy credential-free identity'); + const { requested: requestedRepository, durable: durableRepository } = repositories; + const scrubbedMetadata = state.recoveryMetadata === undefined + ? undefined : scrubDurableRecoveryMetadata(state.recoveryMetadata, this.adapter.provider); + const repositoryNeedsScrub = Boolean(state.activeTurn + && JSON.stringify(state.activeTurn.repository) !== JSON.stringify(durableRepository)); + const metadataNeedsScrub = JSON.stringify(state.recoveryMetadata) !== JSON.stringify(scrubbedMetadata); + if (repositoryNeedsScrub || metadataNeedsScrub) { + state = await this.compareAndSetExact(state, { + activeTurn: repositoryNeedsScrub + ? { ...state.activeTurn!, repository: durableRepository } : state.activeTurn, + recoveryMetadata: scrubbedMetadata, + }, 'A newer operation superseded durable security scrubbing'); + } + const durableFingerprint = fingerprintGoalWorktree(durableRepository); + if (fingerprintGoalWorktree(requestedRepository) !== durableFingerprint) return this.blockRecovery(fence, state, + 'Requested worktree does not match the active turn\'s authoritative repository identity'); + const container = sanitizeContainerInspection(await this.ports.recovery.inspectContainer({ + goalId: identity.goalId, sessionId: identity.sessionId, + })); + state = await this.revalidateInspectionState(state, fence); + const rawRepositoryInspection = await this.ports.recovery.inspectRepository(durableRepository); + state = await this.revalidateInspectionState(state, fence); + const repositoryInspection = sanitizeRepositoryInspection(durableRepository, rawRepositoryInspection); + const mismatch = verifyReconciliationTarget(durableRepository, repositoryInspection) + ?? verifyRecoveredContainer(state, container, durableFingerprint); + if (mismatch) return this.blockRecovery(fence, state, mismatch); + return { state, fence, container, repository: repositoryInspection }; + } + + private async blockRecovery(fence: GoalSessionControlFence, state: GoalSessionState, reason: string): + Promise { + try { + const saved = await this.commitControlTransition({ + state, + fence, + changes: {}, + auditEvents: [{ type: 'reconciliation', outcome: 'blocked', reason }], + transitionId: `${this.controlOperationId('recovery-blocked', state)}:${reason}`, + execution: controlExecutionIdentity(state), + }); + return { outcome: 'blocked', reason, state: saved }; + } catch (error) { + if (!(error instanceof StaleGoalSessionFenceError)) throw error; + const current = await this.requireState(fence); + const guarded = await this.guardReconciliationState(current, fence); + if (guarded) return guarded; + throw error; + } + } + + private async handleRecoveryPromotionLoss(error: unknown, identity: GoalSessionIdentity, fence: GoalSessionControlFence): + Promise { + if (!(error instanceof StaleGoalSessionFenceError)) throw error; + const state = await this.requireState(identity); + const cancelled = await this.guardReconciliationState(state, fence); + if (cancelled) return cancelled; + throw error; + } + + private async persistRecoveryResult(fence: GoalSessionControlFence, state: GoalSessionState, + execution: GoalExecutionIdentity, result: Awaited>): + Promise { + const snapshot = 'snapshot' in result ? result.snapshot : undefined; + const reason = safeFailureDiagnostic(result.reason, 'Provider reconciliation completed safely'); + if (snapshot) { + assertProviderIdentity(state, snapshot); + assertCredentialFreeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider); + } + const reconciled = reconcileRecoveredTurn(state, execution, result.outcome); + if (result.outcome === 'failed') { + const saved = await this.ports.terminal.commit(state, nextState(state, { + status: 'failed', activeTurn: undefined, recoveryAttempt: undefined, + resumeIntent: undefined, completedResume: undefined, + completedRecovery: { + operationToken: state.recoveryAttempt!.operationToken, + controllerEpoch: fence.controllerEpoch, outcome: 'failed', reason, + }, + failureReason: reason, initializationIntent: undefined, retryTurn: undefined, + pendingAfterTurnPause: undefined, pendingModelChange: undefined, + modelChangeIntent: undefined, modelChangeIntents: undefined, + providerOperationGeneration: (state.providerOperationGeneration ?? 0) + 1, + }), { + scope: 'control', fence, execution, + auditEvents: [{ type: 'reconciliation', outcome: 'failed', reason }], + event: { type: 'completion', outcome: 'failed', error: reason }, + }); + if (!saved) throw new StaleGoalSessionFenceError('A newer operation superseded failed reconciliation'); + return { outcome: 'failed', reason, state: saved }; + } + const preserveIntentModel = this.adapter.capabilities.modelChange === 'next_safe_boundary' + ? hasUnresolvedImmediateModelIntent(state) + : latestImmediateModelIntent(state)?.invocationEvidence !== undefined; + const recoveredModelEvidence = result.outcome === 'resumed' ? prepareModelEvidenceForRecoveredAttempt(state, execution) : {}; + let saved: GoalSessionState; + try { + saved = await this.commitControlTransition({ + state, + fence, + changes: { + status: reconciled.status, + activeTurn: reconciled.activeTurn, + ...recoveredModelEvidence, + recoveryAttempt: undefined, + completedRecovery: { + operationToken: state.recoveryAttempt!.operationToken, + controllerEpoch: fence.controllerEpoch, + outcome: result.outcome, + reason, + }, + failureReason: undefined, + providerSessionId: snapshot?.providerSessionId ?? state.providerSessionId, + recoveryMetadata: snapshot + ? sanitizeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider) + : state.recoveryMetadata === undefined + ? undefined : sanitizeRecoveryMetadata(state.recoveryMetadata, this.adapter.provider), + currentModel: preserveIntentModel ? state.currentModel : snapshot?.model ?? state.currentModel, + }, + auditEvents: [{ type: 'reconciliation', outcome: result.outcome, reason }], + transitionId: `recovery-result:${state.recoveryAttempt!.operationToken}`, + execution, + }); + } catch (error) { + await this.expireRecoveryLeaseIfOwned(fence, state.recoveryAttempt!.operationToken); + throw error; + } + const recovered = await this.resumeImmediateModelChangeIntent(fence, saved); + return { outcome: result.outcome, reason, state: recovered }; + } + + private async guardReconciliationState(state: GoalSessionState, fence: GoalSessionControlFence): + Promise { + if (state.status === 'terminated' || state.status === 'failed') { + return { outcome: 'blocked', reason: `A ${state.status} session cannot be reconciled`, state }; + } + if (state.status !== 'cancelling') { + if (isRecoverableStatus(state.status)) return null; + return { outcome: 'blocked', reason: `A ${state.status} session cannot be reconciled`, state }; + } + const cancelled = state.cancellationIntent + ? await this.resumeClaimedCancellation(fence, state) + : await this.cancel({ + ...fence, + reason: state.failureReason ?? 'Resume pending cancellation during reconciliation', + }); + return { + outcome: 'blocked', + reason: 'Cancellation recovery completed without reconciling provider work', + state: cancelled, + }; + } + + private async claimRecoveryAttempt(state: GoalSessionState, controllerEpoch: number): + Promise<{ state: GoalSessionState; execution: GoalExecutionIdentity } | null> { + assertRecoverableExactState(state, controllerEpoch); + if (Date.parse(state.recoveryAttempt?.leaseExpiresAt ?? '') > Date.now()) return null; + const previousAttempt = state.recoveryAttempt?.attemptId + ?? state.recoveryAttemptId + ?? state.activeTurn?.attemptId + ?? state.providerOpenAttemptId; + const attemptId = previousAttempt ? this.mintFreshAttemptId(previousAttempt) : this.mintAttemptId(); + const execution = { + executionId: state.activeTurn?.executionId ?? `reconcile-${state.sessionId}`, + attemptId, + }; + const operationGeneration = (state.providerOperationGeneration ?? 0) + 1; + const saved = await this.compareAndSetExact(state, { + providerOperationGeneration: operationGeneration, + recoveryAttemptId: attemptId, + completedRecovery: undefined, + recoveryAttempt: { + operationToken: this.controlOperationId('recovery-provider', state), + operationGeneration, + ...execution, + controllerEpoch, + authoritativeAttemptId: state.activeTurn?.attemptId, + authoritativeExecutionId: state.activeTurn?.executionId, + sessionStatus: state.status, + authoritativeTurnStatus: state.activeTurn?.status, + claimedAt: nowIso(), + leaseExpiresAt: new Date(Date.now() + RECOVERY_LEASE_MS).toISOString(), + phase: 'claimed', + }, + }, 'A newer operation superseded crash reconciliation'); + return { state: saved, execution }; + } + + private promoteRecoveryAttempt(state: GoalSessionState, execution: GoalExecutionIdentity, controllerEpoch: number): + Promise { + assertRecoverableExactState(state, controllerEpoch); + if (state.recoveryAttempt?.attemptId !== execution.attemptId + || state.recoveryAttempt.executionId !== execution.executionId + || state.recoveryAttempt.controllerEpoch !== controllerEpoch + || state.recoveryAttempt.phase === 'provider_in_doubt') { + throw new StaleGoalSessionFenceError('Reconciliation no longer owns its provider-call lease'); + } + return this.compareAndSetExact(state, { + recoveryAttempt: { ...state.recoveryAttempt, phase: 'provider_in_doubt' }, + }, 'Cancellation fenced reconciliation before its provider call'); + } + + private async expireRecoveryLeaseIfOwned(fence: GoalSessionControlFence, operationToken: string): Promise { + await expireRecoveryLease({ + ports: this.ports, fence, operationToken, + load: () => this.requireControlledStateForBarrier(fence), + publish: generation => this.publishProviderOperationBarrier(fence, generation), + }); + } + + private revalidatePreparedRecovery(prepared: PreparedRecovery): Promise { + return this.revalidateInspectionState(prepared.state, prepared.fence); + } + + private async committedRecoveryResult(identity: GoalSessionIdentity, controllerEpoch: number): + Promise { + const state = await this.requireState(identity); + return completedRecoveryResult(state, controllerEpoch); + } + + private async revalidateInspectionState(expected: GoalSessionState, fence: GoalSessionControlFence): + Promise { + return revalidateRecoveryInspection({ + expected, fence, + load: () => this.requireControlledState(fence), + guard: state => this.guardReconciliationState(state, fence), + }); + } + + private async requireLiveRecoveryLease(fence: GoalSessionControlFence, execution: GoalExecutionIdentity, + operationToken: string): Promise { + const state = await this.requireControlledState(fence); + assertLiveRecoveryLease(state, execution, operationToken); + return state; + } +} diff --git a/packages/core/src/agents/goalSession/GoalSessionSupervisor.ts b/packages/core/src/agents/goalSession/GoalSessionSupervisor.ts new file mode 100644 index 000000000..96b0f1d43 --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalSessionSupervisor.ts @@ -0,0 +1,398 @@ +import type { GoalSessionState } from './contract.js'; +import { isDeepStrictEqual } from 'node:util'; +import { + GoalSessionContractError, + GoalSessionScopeError, + StaleGoalSessionFenceError, + UnsupportedGoalSessionTransitionError, +} from './errors.js'; +import { createFirstTurnInitializationIntent, deterministicOpenKey, firstTurnIdentityFailure } from './firstTurnIdentity.js'; +import { decodeDurableGoalSessionState } from './durableStateSecurity.js'; +import { GoalSessionRecoveryControls } from './GoalSessionRecoveryControls.js'; +import { + compactImmediateModelIntents, + hasUnresolvedImmediateModelIntent, + immediateModelIntents, + latestImmediateModelIntent, +} from './modelChangeProtocol.js'; +import { assertCredentialFreeRecoveryMetadata, sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +import { safeFailureDiagnostic } from './securityBoundary.js'; +import { rebuildProviderSnapshot } from './providerResultBoundary.js'; +import { throwPersistedProviderOpenFailure } from './providerOpenFailure.js'; +import { credentialFreeRepositoryIdentity } from './repositorySecurity.js'; +import { + createOptionalClaimedOpenContext, durableCodexOpenKey, validateSupervisedOpenPlan, + type GoalOwnedOpenContext, type OpenGoalSessionRequest, +} from './goalSessionOpen.js'; +import { + assertProviderIdentity, + nextState, + nowIso, + persistedSnapshot, + validateEpoch, + validateIdentity, +} from './support.js'; + +export type { + GoalSupervisedOpenClaim, GoalSupervisedOpenPlan, OpenGoalSessionRequest, +} from './goalSessionOpen.js'; + +export type { ReconcileGoalSessionResult } from './GoalSessionRecoveryControls.js'; + +/** + * Coordinates durable goal turns. The class has no dependency on API routes or + * queue implementations; callers inject the goal persistence/event/message ports. + * Turn execution lives in {@link GoalTurnRunner}; this layer owns session open, + * crash recovery, and the session-scoped control operations. + */ +export class GoalSessionSupervisor extends GoalSessionRecoveryControls { + async openSession(request: OpenGoalSessionRequest): Promise { + validateIdentity(request); + validateEpoch(request.controllerEpoch); + if (request.provider !== this.adapter.provider) { + throw new GoalSessionContractError( + `Adapter "${this.adapter.provider}" cannot open provider "${request.provider}"`, + 'UNSUPPORTED_PROVIDER', + ); + } + if (request.supervisedOpen) await validateSupervisedOpenPlan(this.adapter, request.supervisedOpen); + if (request.provider === 'codex' && this.adapter.capabilities.nativeSessionId === 'eager' + && !request.supervisedOpen) throw new GoalSessionContractError( + 'Eager Codex open requires a post-claim supervised factory', 'OPEN_CONTEXT_MISSING', + ); + request = { + goalId: request.goalId, sessionId: request.sessionId, provider: request.provider, + controllerEpoch: request.controllerEpoch, supervisedOpen: request.supervisedOpen, + }; + + const opened = await this.loadOrCreateForOpen(request); + let state = opened.state; + state = await this.scrubDurableSecurityState(state); + state = await this.repairPendingProviderBarrier({ + goalId: request.goalId, sessionId: request.sessionId, controllerEpoch: state.controllerEpoch, + }, state); + if (state.status === 'terminated') { + if (state.cancellationIntent) return state; + throw new GoalSessionContractError('A terminated provider session cannot be resumed', 'SESSION_TERMINATED'); + } + if (state.status === 'failed' && this.adapter.capabilities.nativeSessionId === 'eager') { + throw new GoalSessionContractError('A failed provider session cannot be resumed', 'SESSION_TERMINATED'); + } + if (state.status === 'cancelling') { + if (!state.cancellationIntent) { + return this.cancel({ + goalId: request.goalId, + sessionId: request.sessionId, + controllerEpoch: state.controllerEpoch, + reason: state.failureReason ?? 'Resume pending cancellation after process replacement', + }); + } + return this.resumeClaimedCancellation({ + goalId: request.goalId, + sessionId: request.sessionId, + controllerEpoch: state.controllerEpoch, + }, state); + } + if (request.controllerEpoch > state.controllerEpoch) state = await this.takeover(request, request.controllerEpoch); + state = await this.compactModelIntentRetention(state); + + let deterministicOpenKey: string | undefined; + if (!state.providerSessionId) { + if (this.adapter.capabilities.nativeSessionId === 'first_turn') { + return this.openFirstTurnIdentitySession(request, state); + } + if (!opened.created && !this.canRecoverIncompleteInit(state)) { + throw new GoalSessionContractError( + 'The previous controller stopped before persisting a provider session identity; reconcile or fail this goal explicitly', + 'INCOMPLETE_INITIALIZATION', + ); + } + state = await this.recordInitializationIntent(request, state); + deterministicOpenKey = state.initializationIntent?.deterministicOpenKey; + } else { + state = await this.recordProviderOpenAttempt(state); + } + + state = await this.callProviderOpen(request, state, deterministicOpenKey); + return this.resumeImmediateModelChangeIntent(request, state); + } + + private async compactModelIntentRetention(state: GoalSessionState): Promise { + const intents = immediateModelIntents(state); + for (const intent of intents) { + const historical = await this.ports.modelChanges.claim(state, intent.modelChangeId, intent.model); + if (historical.model !== intent.model) { + throw new GoalSessionContractError( + 'Durable model history conflicts with retained session state', 'MODEL_OPERATION_CONFLICT', + ); + } + if ((intent.phase === 'committed' || intent.phase === 'superseded') && !intent.applicationToken) { + await this.ports.modelChanges.settle(state, intent.modelChangeId, intent.acknowledgement ?? { + requestedModel: intent.model, + appliesAt: this.adapter.capabilities.modelChange === 'next_turn' ? 'next_turn' : 'next_safe_boundary', + effectiveModel: intent.phase === 'committed' ? intent.model : undefined, + }); + } + } + const compacted = compactImmediateModelIntents(intents); + if (compacted.length === intents.length) return state; + return this.compareAndSetExact(state, { + modelChangeIntents: compacted, + modelChangeIntent: compacted.at(-1), + }, 'A newer operation superseded model intent retention during reopen'); + } + + private async scrubDurableSecurityState(state: GoalSessionState): Promise { + // requireState/loadOrCreate already rebuilt every field. Security + // normalization is validation-only on reopen: corruption must leave no + // mutation trace and cancellation identity is never synthesized. + const decoded = decodeDurableGoalSessionState(state); + if (decoded.activeTurn) { + const repository = await credentialFreeRepositoryIdentity(decoded.activeTurn.repository); + if (!isDeepStrictEqual(repository, decoded.activeTurn.repository)) { + throw new GoalSessionContractError('Durable repository identity is not canonical', 'INVALID_DURABLE_STATE'); + } + } + if (decoded.failureReason !== undefined + && safeFailureDiagnostic(decoded.failureReason, 'Provider operation failed safely') !== decoded.failureReason) { + throw new GoalSessionContractError('Durable failure reason is unsafe', 'INVALID_DURABLE_STATE'); + } + return decoded; + } + + private canRecoverIncompleteInit(state: GoalSessionState): boolean { + return this.adapter.supportsDeterministicOpen === true && state.initializationIntent !== undefined; + } + + private async openFirstTurnIdentitySession( + request: OpenGoalSessionRequest, + state: GoalSessionState, + ): Promise { + const policy = this.adapter.capabilities.nativeSessionId === 'first_turn' + ? this.adapter.capabilities.firstTurnIdCrashPolicy + : 'fail'; + if (policy === 'fail' && await this.cleanAlreadyTerminalFirstTurn(state)) throw firstTurnIdentityFailure(policy); + if (!state.initializationIntent) { + throw new GoalSessionContractError( + 'A first-turn provider has no durable initialization intent; refusing to start a different native session', + 'INCOMPLETE_INITIALIZATION', + ); + } + if (state.activeTurn && policy === 'retry_deterministically') { + const crashedTurn = state.activeTurn; + return this.updateControlledState(request, value => ({ + ...value, + status: 'idle', + retryTurn: value.activeTurn ? { + turnId: value.activeTurn.turnId, + executionId: value.activeTurn.executionId, + crashedAttemptId: value.activeTurn.attemptId, + } : undefined, + activeTurn: undefined, + completedTurnIds: value.completedTurnIds.filter(turnId => turnId !== crashedTurn.turnId), + completedTurns: value.completedTurns?.filter(turn => turn.turnId !== crashedTurn.turnId), + initializationIntent: value.initializationIntent ? { + attemptId: this.mintFreshAttemptId(value.initializationIntent.attemptId), + deterministicOpenKey: value.initializationIntent.deterministicOpenKey, + recordedAt: nowIso(), + } : value.initializationIntent, + failureReason: undefined, + })); + } + if (state.activeTurn || (state.status !== 'initializing' && state.status !== 'idle')) { + if (policy === 'fail' && state.activeTurn) { + const turn = state.activeTurn; + const completedTurns = state.completedTurns?.some(value => value.turnId === turn.turnId) + ? state.completedTurns + : [...(state.completedTurns ?? []), { + turnId: turn.turnId, executionId: turn.executionId, attemptId: turn.attemptId, + }]; + const failure = firstTurnIdentityFailure(policy); + const saved = await this.ports.terminal.commit(state, nextState(state, { + status: 'failed', activeTurn: undefined, initializationIntent: undefined, retryTurn: undefined, + completedTurnIds: state.completedTurnIds.includes(turn.turnId) + ? state.completedTurnIds : [...state.completedTurnIds, turn.turnId], + completedTurns, failureReason: failure.message, + }), { + scope: 'turn', fence: { ...request, turnId: turn.turnId }, + execution: { executionId: turn.executionId, attemptId: turn.attemptId }, + auditEvents: [], + event: { type: 'completion', outcome: 'failed', error: failure.message }, + }); + if (!saved) throw new StaleGoalSessionFenceError('A newer operation superseded first-turn crash failure'); + if (saved.activeTurn) { + await this.compareAndSetExact(saved, { + status: 'failed', activeTurn: undefined, initializationIntent: undefined, + retryTurn: undefined, failureReason: failure.message, + }); + } + throw failure; + } + throw firstTurnIdentityFailure(policy); + } + if (state.status === 'idle') return state; + return this.compareAndSetExact(state, { status: 'idle', failureReason: undefined }, + 'A newer operation superseded lazy provider initialization'); + } + + private async cleanAlreadyTerminalFirstTurn(state: GoalSessionState): Promise { + if (state.status !== 'failed') return false; + if (!state.activeTurn) return true; + if (state.activeTurn.status !== 'failed') return false; + await this.compareAndSetExact(state, { + activeTurn: undefined, + initializationIntent: undefined, + retryTurn: undefined, + }, 'A newer operation superseded terminal first-turn cleanup'); + return true; + } + + private async recordInitializationIntent( + request: OpenGoalSessionRequest, + state: GoalSessionState, + ): Promise { + // Reopen must first replay the exact settled open operation. Minting a + // fresh attempt here would bypass that receipt and issue a second + // Codex thread/start after a response-before-state-CAS crash. + if (state.initializationIntent) return state; + const attemptId = this.mintAttemptId(); + const operationGeneration = (state.providerOperationGeneration ?? 0) + 1; + return this.compareAndSetExact(state, { + initializationIntent: { + attemptId, + deterministicOpenKey: deterministicOpenKey(request), + recordedAt: nowIso(), + }, + providerOpenAttemptId: attemptId, + providerOpenOperationGeneration: operationGeneration, + providerOperationGeneration: operationGeneration, + }); + } + + private recordProviderOpenAttempt(state: GoalSessionState): Promise { + const attemptId = state.providerOpenAttemptId + ? this.mintFreshAttemptId(state.providerOpenAttemptId) + : this.mintAttemptId(); + const operationGeneration = (state.providerOperationGeneration ?? 0) + 1; + return this.compareAndSetExact(state, { + providerOpenAttemptId: attemptId, + providerOpenOperationGeneration: operationGeneration, + providerOperationGeneration: operationGeneration, + }); + } + + private async loadOrCreateForOpen(request: OpenGoalSessionRequest): Promise<{ state: GoalSessionState; created: boolean }> { + let loaded: GoalSessionState | null; + try { loaded = await this.ports.state.load(request); } + catch (error) { + if (!(error instanceof GoalSessionScopeError)) throw error; + loaded = null; + } + let state = loaded ? decodeDurableGoalSessionState(loaded) : null; + let created = false; + if (!state) { + const timestamp = nowIso(); + const initializationIntent = this.adapter.capabilities.nativeSessionId === 'first_turn' + ? createFirstTurnInitializationIntent(request, this.mintAttemptId()) + : undefined; + const initial = await this.ports.state.create({ + goalId: request.goalId, + sessionId: request.sessionId, + provider: request.provider, + controllerEpoch: request.controllerEpoch, + status: 'initializing', + completedTurnIds: [], + initializationIntent, + createdAt: timestamp, + updatedAt: timestamp, + }); + if (initial) { state = decodeDurableGoalSessionState(initial); created = true; } + else state = await this.requireState(request); + } + if (state.provider !== request.provider) { + throw new UnsupportedGoalSessionTransitionError('A session cannot change providers while resuming', 'UNSUPPORTED_PROVIDER_TRANSITION'); + } + if (request.controllerEpoch < state.controllerEpoch) throw new StaleGoalSessionFenceError(); + return { state, created }; + } + + private async callProviderOpen( + request: OpenGoalSessionRequest, + state: GoalSessionState, + deterministicOpenKey: string | undefined, + ): Promise { + const persisted = state.providerSessionId ? persistedSnapshot(state) : undefined; + let claimedOpen: GoalOwnedOpenContext | undefined; + try { + if (!state.providerOpenAttemptId) { + throw new GoalSessionContractError('Provider open attempt was not durably claimed', 'OPEN_ATTEMPT_MISSING'); + } + const providerOpenAttemptId = state.providerOpenAttemptId; + const operationGeneration = state.providerOpenOperationGeneration + ?? state.providerOperationGeneration ?? 0; + await this.publishProviderOperationBarrier(request, operationGeneration); + const operationFence = this.providerOperationFence( + request, operationGeneration, { kind: 'open', operationId: providerOpenAttemptId }, + ); + const executionId = this.controlOperationId('open-execution', state); + claimedOpen = await createOptionalClaimedOpenContext({ + adapter: this.adapter, plan: request.supervisedOpen, executionId, + attemptId: providerOpenAttemptId, openKey: deterministicOpenKey ?? durableCodexOpenKey(state), + operationGeneration, + operationFence: this.providerOperationFence(request, operationGeneration, { + kind: 'open', operationId: providerOpenAttemptId, + executionId, attemptId: providerOpenAttemptId, + }), + requireCurrent: () => this.requireProviderGeneration(request, operationGeneration), + }); + const openContext = claimedOpen?.context; + const authoritative = await this.requireProviderGeneration(request, operationGeneration); + if (authoritative.providerOpenAttemptId !== providerOpenAttemptId) { + throw new StaleGoalSessionFenceError('Provider open claim was durably replaced'); + } + const effectiveOpenKey = deterministicOpenKey ?? openContext?.deterministicOpenKey; + const snapshot = await this.providerResult(() => this.providerFirstEffect(operationFence, () => { + const completion = this.adapter.openSession({ + goalId: request.goalId, sessionId: request.sessionId, + provider: request.provider, controllerEpoch: request.controllerEpoch, persisted, + deterministicOpenKey: effectiveOpenKey, + attemptId: providerOpenAttemptId, operationGeneration, operationFence, + openContext: openContext ? { + ...openContext, + deterministicOpenKey: effectiveOpenKey, + } : undefined, + }); + return this.startedProviderEffect( + completion, + () => openContext?.transport.cancel() ?? this.rollbackProviderPrimitive(operationFence, state), + ); + }, value => rebuildProviderSnapshot(value, this.adapter.provider)), + value => rebuildProviderSnapshot(value, this.adapter.provider)); + assertCredentialFreeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider); + assertProviderIdentity(state, snapshot); + const preserveIntentModel = this.adapter.capabilities.modelChange === 'next_safe_boundary' + ? hasUnresolvedImmediateModelIntent(state) + : latestImmediateModelIntent(state)?.invocationEvidence !== undefined; + const saved = await this.ports.state.compareAndSet(state, nextState(state, { + providerSessionId: snapshot.providerSessionId, + recoveryMetadata: sanitizeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider), + currentModel: preserveIntentModel ? state.currentModel : snapshot.model ?? state.currentModel, + status: state.status === 'initializing' ? 'idle' : state.status, + initializationIntent: undefined, + failureReason: undefined, + })); + if (!saved) throw new StaleGoalSessionFenceError('Session ownership changed while provider identity was being persisted'); + claimedOpen?.transfer(); + return saved; + } catch (error) { + await claimedOpen?.cancel().catch(() => undefined); + return throwPersistedProviderOpenFailure(this.ports.state, state, error); + } + } + +} + +export { GoalSessionContractError, StaleGoalSessionFenceError, UnsupportedGoalSessionTransitionError } from './errors.js'; +export { assertCredentialFreeRecoveryMetadata } from './recoveryMetadata.js'; +export { firstPendingCorrectiveMessage } from './support.js'; +export type { RunGoalTurnRequest, RunGoalTurnResult } from './GoalTurnRunner.js'; diff --git a/packages/core/src/agents/goalSession/GoalTurnRunner.ts b/packages/core/src/agents/goalSession/GoalTurnRunner.ts new file mode 100644 index 000000000..a0590c826 --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalTurnRunner.ts @@ -0,0 +1,412 @@ +import type { GoalBeginTurnRequest, GoalExecutionIdentity, GoalModelChangeIntent, GoalProviderCorrectiveMessage, GoalSessionControlFence, GoalSessionFence, GoalSessionState, GoalTurnResumeCapabilityOutcome } from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { GoalTurnStreamRunner } from './GoalTurnStreamRunner.js'; +import { assertCredentialFreeRecoveryMetadata, sanitizeNewRecoveryMetadata } from './recoveryMetadata.js'; +import { credentialFreeRepositoryIdentity, validateTurnRequestIdentity } from './repositorySecurity.js'; +import { assertProviderIdentity, nextState, persistedSnapshot, providerTurnContext, validateControlFence } from './support.js'; +import { duplicateTurnResult, type RunGoalTurnResult } from './turnDelivery.js'; +import { assertSafeProviderIdentifier, safeDiagnostic } from './securityBoundary.js'; +import { compactImmediateModelIntents, immediateModelIntents, nextModelGeneration } from './modelChangeProtocol.js'; +import { rebuildProviderSnapshot } from './providerResultBoundary.js'; +import { resolveDeferredModel, settledResumeKind, turnExecution } from './turnExecutionProtocol.js'; + +export interface RunGoalTurnRequest extends Omit { + executionId: string; attemptId?: string; +} + +export type { RunGoalTurnResult } from './turnDelivery.js'; + +export abstract class GoalTurnRunner extends GoalTurnStreamRunner { + async runTurn(request: RunGoalTurnRequest): Promise { + validateControlFence(request); + validateTurnRequestIdentity(request); + assertSafeProviderIdentifier(request.requestedModel); + if (request.context !== undefined) assertCredentialFreeRecoveryMetadata(request.context, this.adapter.provider); + const safeRequest: RunGoalTurnRequest = { + goalId: request.goalId, sessionId: request.sessionId, + controllerEpoch: request.controllerEpoch, turnId: request.turnId, + executionId: request.executionId, attemptId: request.attemptId, + objective: safeDiagnostic(request.objective, '[redacted objective]'), + context: request.context === undefined ? undefined : sanitizeNewRecoveryMetadata(request.context, this.adapter.provider), + repository: await credentialFreeRepositoryIdentity(request.repository), + requestedModel: safeDiagnostic(request.requestedModel, 'default'), + }; + let state = await this.requireControlledState(safeRequest); + const execution = turnExecution( + state, safeRequest, + () => this.mintAttemptId(), previous => this.mintFreshAttemptId(previous), + ); + + const duplicate = duplicateTurnResult(state, safeRequest.turnId, execution); + if (duplicate) return duplicate; + if (state.status !== 'idle') { + throw new GoalSessionContractError(`Cannot begin a turn while session is ${state.status}`, 'SESSION_NOT_IDLE'); + } + + state = await this.claimImplicitNextTurnModel(safeRequest, state); + + const { requestedModel, activeModelChange, providerModelChange } = resolveDeferredModel( + state, safeRequest.requestedModel, this.adapter.capabilities.modelChange === 'next_turn', + ); + assertSafeProviderIdentifier(requestedModel); + const correctiveMessages = await this.nextTurnCorrectiveMessages(safeRequest); + const operationGeneration = (state.providerOperationGeneration ?? 0) + 1; + const activeTurn = { + ...execution, + turnId: safeRequest.turnId, + executionEpoch: safeRequest.controllerEpoch, + objective: safeRequest.objective, + requestedModel, + repository: safeRequest.repository, + providerOperationGeneration: operationGeneration, + modelChange: activeModelChange, + status: 'running' as const, + }; + const claimed = await this.ports.state.compareAndSet(state, nextState(state, { + activeTurn, + requestedModel, + status: 'running', + providerOperationGeneration: operationGeneration, + retryTurn: undefined, + modelChangeIntent: state.modelChangeIntent, + })); + if (!claimed) { + state = await this.requireControlledState(safeRequest); + const redelivery = duplicateTurnResult(state, safeRequest.turnId, execution); + if (redelivery) return redelivery; + throw new StaleGoalSessionFenceError('Another delivery claimed the session turn'); + } + + const adapterRequest: GoalBeginTurnRequest = { + ...safeRequest, + ...execution, + requestedModel, + correctiveMessages: correctiveMessages.length ? correctiveMessages : undefined, + operationGeneration, + operationFence: this.turnProviderOperationFence(safeRequest, execution, operationGeneration), + modelChange: providerModelChange, + }; + const outcome = await this.driveTurnStream({ + fence: safeRequest, + execution, + initial: claimed, + nextTurnMessages: correctiveMessages, + openStream: async () => { + await this.publishProviderOperationBarrier(safeRequest, operationGeneration); + await this.requireTurnProviderGeneration(safeRequest, execution, operationGeneration); + return this.providerFirstEffectStream( + adapterRequest.operationFence, + () => this.adapter.beginTurn(adapterRequest, providerTurnContext(claimed)), + ); + }, + }); + return { disposition: 'started', state: outcome.state, execution }; + } + + private async claimImplicitNextTurnModel( + request: GoalSessionControlFence & { requestedModel: string }, + state: GoalSessionState, + ): Promise { + if (this.adapter.capabilities.modelChange !== 'next_turn' + || state.pendingModelChange !== undefined + || state.currentModel === undefined + || state.currentModel === request.requestedModel) return state; + const modelChangeId = this.controlOperationId('model', state); + const historical = await this.ports.modelChanges.claim(request, modelChangeId, request.requestedModel); + if (historical.model !== request.requestedModel) { + throw new GoalSessionContractError('Implicit model identity conflicts with durable history', 'MODEL_OPERATION_CONFLICT'); + } + const generation = nextModelGeneration(state); + const intent: GoalModelChangeIntent = { + modelChangeId, model: request.requestedModel, requestedAt: new Date().toISOString(), + generation, previousModel: state.currentModel, phase: 'pending', + }; + return this.commitControlTransition({ + state, fence: request, + changes: { + requestedModel: request.requestedModel, pendingModelChange: request.requestedModel, + modelChangeIntent: intent, + modelChangeIntents: compactImmediateModelIntents([...immediateModelIntents(state), intent]), + modelChangeGeneration: generation, + }, + auditEvents: [{ + type: 'model_change_acknowledged', requestedModel: request.requestedModel, appliesAt: 'next_turn', + }], + transitionId: `model-requested:${modelChangeId}`, + }); + } + + private async nextTurnCorrectiveMessages( + request: GoalSessionControlFence, + ): Promise { + if (this.adapter.capabilities.steering !== 'next_turn') return []; + const pending = await this.ports.messages.listPending(request); + return pending + .sort((left, right) => left.sequence - right.sequence) + .map(({ messageId, sequence, body }) => ({ messageId, sequence, body })); + } + + async resumeTurn(fence: GoalSessionControlFence): Promise { + const state = await this.requireControlledState(fence); + if (settledResumeKind(state, 'recovered_after_turn')) { + return this.continueSettledRecoveredAfterTurn(fence, state); + } + if (settledResumeKind(state, 'active_turn')) { + return this.continueSettledActiveResume(fence, state); + } + if (this.adapter.capabilities.pause === 'after_turn') { + return this.resumeAfterTurn(fence, state); + } + return this.resumeActiveTurn(fence, state); + } + + private resumeAfterTurn( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise | GoalTurnResumeCapabilityOutcome { + if (state.status === 'paused' && state.activeTurn?.status === 'paused' + && state.recoveryAttemptId === state.activeTurn.attemptId) { + return this.retryRecoveredAfterTurn(fence, state); + } + return { disposition: 'unsupported_same_turn', supportedBoundary: 'after_turn' }; + } + + private async resumeActiveTurn( + fence: GoalSessionControlFence, + initial: GoalSessionState, + ): Promise { + let state = initial; + if (state.status !== 'paused' || !state.activeTurn || state.activeTurn.status !== 'paused') { + throw new GoalSessionContractError(`Cannot resume a turn while the session is ${state.status}`, 'SESSION_NOT_PAUSED'); + } + const previousAttemptId = state.activeTurn.attemptId; + const execution: GoalExecutionIdentity = { + executionId: state.activeTurn.executionId, + attemptId: this.mintFreshAttemptId(previousAttemptId), + }; + const turnFence: GoalSessionFence = { ...fence, turnId: state.activeTurn.turnId }; + + state = await this.claimResumeOperation(fence, state, { + kind: 'active_turn', execution, turnId: turnFence.turnId, + }); + state = await this.compareAndSetExact(state, { + activeTurn: { + ...state.activeTurn!, ...execution, executionEpoch: fence.controllerEpoch, + providerOperationGeneration: state.resumeIntent!.operationGeneration, status: 'paused', + }, + }, 'A newer operation replaced the claimed paused turn'); + state = await this.promoteResumeOperation(fence, state); + const intent = state.resumeIntent!; + state = await this.requireLiveResumeOperation(fence, intent.operationId, intent.operationGeneration); + const providerRequest = this.providerResumeRequest(fence, intent); + let snapshot; + try { + await this.publishProviderOperationBarrier(fence, intent.operationGeneration); + await this.requireProviderGeneration(fence, intent.operationGeneration); + snapshot = await this.providerResult( + () => this.providerFirstEffect(providerRequest.operationFence, () => { + const completion = this.adapter.resumeSession(providerRequest, persistedSnapshot(state)); + return this.startedProviderEffect( + completion, + () => this.rollbackProviderPrimitive(providerRequest.operationFence, state), + ); + }, value => rebuildProviderSnapshot(value, this.adapter.provider)), + value => rebuildProviderSnapshot(value, this.adapter.provider), + ); + } catch (error) { + await this.expireResumeOperation(fence, intent.operationId, intent.operationGeneration); + throw error; + } + assertCredentialFreeRecoveryMetadata(snapshot.recoveryMetadata, this.adapter.provider); + assertProviderIdentity(state, snapshot); + state = await this.requireLiveResumeOperation(fence, intent.operationId, intent.operationGeneration); + state = await this.commitControlTransition({ + state, + fence, + changes: { + status: 'running', + activeTurn: { ...state.activeTurn!, ...execution, executionEpoch: fence.controllerEpoch, status: 'running' }, + providerSessionId: snapshot.providerSessionId, + recoveryMetadata: snapshot.recoveryMetadata, + currentModel: snapshot.model ?? state.currentModel, + resumeIntent: { ...intent, phase: 'settled' }, + completedResume: { + operationId: intent.operationId, operationGeneration: intent.operationGeneration, + kind: intent.kind, controllerEpoch: intent.controllerEpoch, + }, + }, + auditEvents: [{ type: 'session_resumed' }, { type: 'turn_resumed', turnId: turnFence.turnId }], + transitionId: `resume-settled:${intent.operationId}:${intent.operationGeneration}`, + execution, + }); + + if (!this.adapter.resumeTurn) { + throw new GoalSessionContractError('Provider declares active-turn pause without implementing turn resume', 'CAPABILITY_METHOD_MISSING'); + } + const resumeTurn = this.adapter.resumeTurn.bind(this.adapter); + const outcome = await this.driveTurnStream({ + fence: turnFence, + execution, + initial: state, + nextTurnMessages: [], + openStream: async () => { + await this.publishProviderOperationBarrier(fence, intent.operationGeneration); + await this.requireTurnProviderGeneration(turnFence, execution, intent.operationGeneration); + return this.providerFirstEffectStream(providerRequest.operationFence, () => + resumeTurn({ ...turnFence, ...execution, ...providerRequest }, persistedSnapshot(state))); + }, + }); + return { disposition: 'started', state: outcome.state, execution }; + } + + private async continueSettledActiveResume( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise { + if (!this.adapter.resumeTurn || !state.activeTurn || !state.resumeIntent) { + throw new GoalSessionContractError('Settled active resume is missing its provider primitive', 'CAPABILITY_METHOD_MISSING'); + } + const turn = state.activeTurn; + const execution = { executionId: turn.executionId, attemptId: turn.attemptId }; + const turnFence = { ...fence, turnId: turn.turnId }; + const providerRequest = this.providerResumeRequest(fence, state.resumeIntent); + const resumeTurn = this.adapter.resumeTurn.bind(this.adapter); + state = await this.requireActiveAttemptState(turnFence, execution); + const outcome = await this.driveTurnStream({ + fence: turnFence, execution, initial: state, nextTurnMessages: [], + openStream: async () => { + await this.publishProviderOperationBarrier(fence, state.resumeIntent!.operationGeneration); + await this.requireTurnProviderGeneration( + turnFence, execution, state.resumeIntent!.operationGeneration, + ); + return this.providerFirstEffectStream(providerRequest.operationFence, () => + resumeTurn({ ...turnFence, ...execution, ...providerRequest }, persistedSnapshot(state))); + }, + }); + return { disposition: 'started', state: outcome.state, execution }; + } + + private async continueSettledRecoveredAfterTurn( + fence: GoalSessionControlFence, + state: GoalSessionState, + ): Promise { + const turn = state.activeTurn!; + const intent = state.resumeIntent!; + const execution = { executionId: turn.executionId, attemptId: turn.attemptId }; + const turnFence = { ...fence, turnId: turn.turnId }; + const correctiveMessages = await this.nextTurnCorrectiveMessages(turnFence); + state = await this.requireActiveAttemptState(turnFence, execution); + const adapterRequest: GoalBeginTurnRequest = { + ...turnFence, ...execution, objective: safeDiagnostic(turn.objective, '[redacted objective]'), + repository: turn.repository, requestedModel: turn.requestedModel, + correctiveMessages: correctiveMessages.length ? correctiveMessages : undefined, + providerOperation: this.providerResumeRequest(fence, intent), + operationGeneration: intent.operationGeneration, + operationFence: this.turnProviderOperationFence(turnFence, execution, intent.operationGeneration), + modelChange: turn.modelChange ? { + modelChangeId: turn.modelChange.modelChangeId, + generation: turn.modelChange.generation, + } : undefined, + }; + const outcome = await this.driveTurnStream({ + fence: turnFence, execution, initial: state, nextTurnMessages: correctiveMessages, + openStream: async () => { + await this.publishProviderOperationBarrier(fence, intent.operationGeneration); + await this.requireTurnProviderGeneration(turnFence, execution, intent.operationGeneration); + return this.providerFirstEffectStream( + adapterRequest.operationFence, + () => this.adapter.beginTurn(adapterRequest, providerTurnContext(state)), + ); + }, + }); + return { disposition: 'started', state: outcome.state, execution }; + } + + /** + * A discrete after-turn provider cannot resume an operator-paused invocation, + * but a reconciled crash retains a paused active turn. Retry that exact logical + * turn through a fresh discrete invocation on the already-bound native session. + */ + private async retryRecoveredAfterTurn(fence: GoalSessionControlFence, state: GoalSessionState): Promise { + const originalTurn = state.activeTurn!; + if (!state.providerSessionId) { + throw new GoalSessionContractError('A crashed after-turn invocation cannot continue before its native session ID is bound', 'FIRST_TURN_ID_NOT_BOUND'); + } + state = await this.requireControlledState(fence); + if (state.status !== 'paused' || state.activeTurn?.turnId !== originalTurn.turnId + || state.activeTurn.attemptId !== originalTurn.attemptId) { + throw new StaleGoalSessionFenceError('A newer operation superseded the recovered turn boundary'); + } + const initialTurn = state.activeTurn; + const execution = { executionId: initialTurn.executionId, attemptId: this.mintFreshAttemptId(initialTurn.attemptId) }; + state = await this.claimResumeOperation(fence, state, { + kind: 'recovered_after_turn', execution, turnId: initialTurn.turnId, + }); + const { requestedModel, activeModelChange, providerModelChange } = resolveDeferredModel( + state, state.activeTurn!.requestedModel, this.adapter.capabilities.modelChange === 'next_turn', + ); + const recoveredModelChange = providerModelChange ?? state.activeTurn!.modelChange; + state = await this.promoteResumeOperation(fence, state); + const intent = state.resumeIntent!; + state = await this.requireLiveResumeOperation(fence, intent.operationId, intent.operationGeneration); + const turn = state.activeTurn!; + const turnFence = { ...fence, turnId: turn.turnId }; + const correctiveMessages = await this.nextTurnCorrectiveMessages(turnFence); + const activeTurn = { + ...turn, + ...execution, + executionEpoch: fence.controllerEpoch, + requestedModel, + modelChange: activeModelChange ?? turn.modelChange, + status: 'running' as const, + providerOperationGeneration: intent.operationGeneration, + }; + const recoveringPause = state.pendingAfterTurnPause === true; + const claimed = await this.commitControlTransition({ + state, + fence, + changes: { + status: recoveringPause ? 'pause_requested' : 'running', + activeTurn: recoveringPause ? { ...activeTurn, status: 'pause_requested' } : activeTurn, + modelChangeIntent: state.modelChangeIntent, + resumeIntent: { ...intent, phase: 'settled' }, + completedResume: { + operationId: intent.operationId, operationGeneration: intent.operationGeneration, + kind: intent.kind, controllerEpoch: intent.controllerEpoch, + }, + }, + auditEvents: [{ type: 'session_resumed' }, { type: 'turn_resumed', turnId: turn.turnId }], + transitionId: `resume-settled:${intent.operationId}:${intent.operationGeneration}`, + execution, + }); + const adapterRequest: GoalBeginTurnRequest = { + ...turnFence, + ...execution, + objective: safeDiagnostic(turn.objective, '[redacted objective]'), + repository: turn.repository, + requestedModel, + correctiveMessages: correctiveMessages.length ? correctiveMessages : undefined, + providerOperation: this.providerResumeRequest(fence, intent), + operationGeneration: intent.operationGeneration, + operationFence: this.turnProviderOperationFence(turnFence, execution, intent.operationGeneration), + modelChange: recoveredModelChange, + }; + const outcome = await this.driveTurnStream({ + fence: turnFence, + execution, + initial: claimed, + nextTurnMessages: correctiveMessages, + openStream: async () => { + await this.publishProviderOperationBarrier(fence, intent.operationGeneration); + await this.requireTurnProviderGeneration(turnFence, execution, intent.operationGeneration); + return this.providerFirstEffectStream( + adapterRequest.operationFence, + () => this.adapter.beginTurn(adapterRequest, providerTurnContext(claimed)), + ); + }, + }); + return { disposition: 'started', state: outcome.state, execution }; + } + +} diff --git a/packages/core/src/agents/goalSession/GoalTurnStreamRunner.ts b/packages/core/src/agents/goalSession/GoalTurnStreamRunner.ts new file mode 100644 index 000000000..1520f3a97 --- /dev/null +++ b/packages/core/src/agents/goalSession/GoalTurnStreamRunner.ts @@ -0,0 +1,389 @@ +import type { + GoalExecutionIdentity, GoalModelChangeIntent, GoalProviderCorrectiveMessage, GoalSessionEvent, + GoalSessionFence, GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { GoalSessionCore } from './GoalSessionCore.js'; +import { assertCredentialFreeRecoveryMetadata, sanitizeNewRecoveryMetadata } from './recoveryMetadata.js'; +import { safeFailureDiagnostic, sanitizeGoalSessionEvent } from './securityBoundary.js'; +import { immediateModelIntents } from './modelChangeProtocol.js'; +import { + assertFirstTurnIdentityEvent, assertSuppliedMessagesAcknowledged, + isAtomicTurnAudit, streamAuditTransitionId, +} from './turnStreamProtocol.js'; +import { rebuildIterator, rebuildIteratorResult } from './providerResultBoundary.js'; + +type TurnStreamOutcome = { state: GoalSessionState; completed: boolean; reachedPause: boolean }; + +interface TurnStreamOptions { + fence: GoalSessionFence; + execution: GoalExecutionIdentity; + initial: GoalSessionState; + nextTurnMessages: GoalProviderCorrectiveMessage[]; + openStream: () => AsyncIterable | Promise>; +} + +interface TurnStreamProgress { + state: GoalSessionState; + completed: boolean; + reachedPause: boolean; + stop: boolean; +} + +/** Exact-attempt stream consumption and atomic event/state persistence. */ +export abstract class GoalTurnStreamRunner extends GoalSessionCore { + protected async driveTurnStream(options: TurnStreamOptions): Promise { + const { fence, execution } = options; + let current = options.initial; + const awaitingMessageIds = options.nextTurnMessages.map(message => message.messageId); + let reachedPause = false; + let completed = false; + try { + const stream = await options.openStream(); + const iterator = await this.providerResult(() => stream[Symbol.asyncIterator](), rebuildIterator); + for (;;) { + const next = await this.providerResult(() => iterator.next(), rebuildIteratorResult); + if (next.done) break; + const progress = await this.processTurnStreamEvent({ + fence, execution, state: current, event: next.value, + awaitingMessageIds, completed, + }); + current = progress.state; + completed = progress.completed; + reachedPause ||= progress.reachedPause; + if (progress.stop) { + if (iterator.return) await this.providerEffect(() => iterator.return!()); + break; + } + } + if (!completed && !reachedPause) { + const error = 'Provider stream ended without a completion or safe pause boundary'; + current = await this.commitTurnCompletion(fence, execution, { type: 'completion', outcome: 'failed', error }); + completed = true; + } + return { state: current, completed, reachedPause }; + } catch (error) { + if (error instanceof StaleGoalSessionFenceError) throw error; + // A persistence/transport crash is not a provider failure and must + // leave the exact durable invocation recoverable. Provider and + // protocol failures have already been rebuilt as contract errors. + if (!(error instanceof GoalSessionContractError)) throw error; + if (recoverableFirstTurnIdentityLoss(error, this.adapter.capabilities)) throw error; + const message = safeFailureDiagnostic((error as Error).message, 'Provider turn failed safely'); + await this.finishTurnIfOwned(fence, execution, message); + // Adapter creation, iterator.next/return, and provider event decoding + // were rebuilt at their exact boundaries above. Anything else was + // raised by trusted runtime persistence and must retain its internal + // contract/crash identity. + throw error; + } + } + + private async processTurnStreamEvent(options: { + fence: GoalSessionFence; + execution: GoalExecutionIdentity; + state: GoalSessionState; + event: GoalSessionEvent; + awaitingMessageIds: string[]; + completed: boolean; + }): Promise { + const { fence, execution, event, awaitingMessageIds } = options; + const consumesModelEvidence = consumesNextTurnModelEvidence( + options.state, event, execution, this.adapter.capabilities.modelChange, + ); + let state = await this.settleNextTurnModelEvidence(fence, execution, options.state, event); + if (consumesModelEvidence) return unchangedStreamProgress(state, options.completed); + if (options.completed) { + throw new GoalSessionContractError('Provider emitted an event after turn completion', 'EVENT_AFTER_COMPLETION'); + } + assertFirstTurnIdentityEvent(state, event, this.adapter.capabilities.nativeSessionId); + if (event.type === 'message_acknowledged') { + await this.acknowledgeNextTurnMessage(fence, execution, event.messageId, awaitingMessageIds); + return unchangedStreamProgress(state, false); + } + assertSuppliedMessagesAcknowledged(event, awaitingMessageIds); + if (event.type === 'completion') assertExactNextTurnModelEvidence(state, execution); + if (event.type === 'completion' && this.adapter.capabilities.pause === 'after_turn') { + state = await this.requireActiveAttemptState(fence, execution); + } + state = await this.applyTurnEvent({ fence, current: state, execution, event }); + if (event.type !== 'completion' && !isAtomicTurnAudit(event)) await this.append(fence, execution, event); + const completed = event.type === 'completion'; + const reachedPause = event.type === 'pause_boundary' || (completed && state.status === 'paused'); + return { + state, completed, reachedPause, + stop: stopsAtActivePause(event, this.adapter.capabilities.pause), + }; + } + + private async acknowledgeNextTurnMessage( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + awaitingMessageIds: string[], + ): Promise { + const expected = awaitingMessageIds[0]; + if (expected !== messageId) { + throw new GoalSessionContractError( + `Provider acknowledged corrective message "${messageId}" while "${expected ?? 'none'}" was next`, + 'MESSAGE_ACK_OUT_OF_ORDER', + ); + } + sanitizeGoalSessionEvent({ type: 'message_acknowledged', messageId }); + const result = await this.ports.messages.acknowledgeWithEvent(fence, execution, messageId); + if (result === 'stale_fence') throw new StaleGoalSessionFenceError(); + if (result === 'not_found') throw new GoalSessionContractError( + 'Corrective message disappeared before acknowledgement', 'MESSAGE_NOT_FOUND', + ); + awaitingMessageIds.shift(); + } + + private async settleNextTurnModelEvidence( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + state: GoalSessionState, + event: GoalSessionEvent, + ): Promise { + if (this.adapter.capabilities.modelChange !== 'next_turn' || !state.activeTurn?.modelChange) return state; + const invocation = state.activeTurn.modelChange; + const durableIntent = immediateModelIntents(state).find(intent => + intent.modelChangeId === invocation.modelChangeId && intent.generation === invocation.generation); + const occurrenceId = invocationEvidenceOccurrence(event); + if (durableIntent?.invocationEvidence) { + const evidence = durableIntent.invocationEvidence; + if (event.type !== 'model_changed') return state; + if (occurrenceId === evidence.occurrenceId && event.model === evidence.effectiveModel + && evidence.executionId === execution.executionId && evidence.attemptId === execution.attemptId) return state; + throw new GoalSessionContractError( + 'Deferred model evidence belongs to a different provider occurrence or attempt', + 'MODEL_EVIDENCE_MISMATCH', + ); + } + if (!occurrenceId) return state; + if (!durableIntent) throw new GoalSessionContractError( + 'Deferred model intent disappeared before invocation evidence', 'MODEL_EVIDENCE_MISSING', + ); + if (event.type !== 'model_changed' || event.model !== durableIntent.model) { + throw new GoalSessionContractError( + 'Provider effective model does not match the deferred model intent', 'MODEL_ACK_MISMATCH', + ); + } + if (sameHistoricalModelOccurrence(durableIntent, event, occurrenceId)) { + const rebound = { + ...durableIntent, + invocationEvidence: modelInvocationEvidence(durableIntent, execution, occurrenceId, event.model), + }; + return this.updateActiveTurnState(fence, execution, value => ({ + currentModel: rebound.model, + pendingModelChange: value.modelChangeIntent?.modelChangeId === rebound.modelChangeId + ? undefined : value.pendingModelChange, + modelChangeIntent: value.modelChangeIntent?.modelChangeId === rebound.modelChangeId + ? rebound : value.modelChangeIntent, + modelChangeIntents: immediateModelIntents(value).map(intent => + intent.modelChangeId === rebound.modelChangeId ? rebound : intent), + })); + } + const acknowledgement = { + outcome: 'acknowledged' as const, + requestedModel: durableIntent.model, + appliesAt: 'next_turn' as const, + effectiveModel: durableIntent.model, + }; + const settled = { + ...durableIntent, + phase: 'committed' as const, + acknowledgement, + invocationEvidence: modelInvocationEvidence(durableIntent, execution, occurrenceId, event.model), + }; + const saved = await this.commitTurnTransition({ + state, fence, execution, + update: value => ({ + currentModel: settled.model, + pendingModelChange: value.modelChangeIntent?.modelChangeId === settled.modelChangeId + ? undefined : value.pendingModelChange, + modelChangeIntent: value.modelChangeIntent?.modelChangeId === settled.modelChangeId + ? settled : value.modelChangeIntent, + modelChangeIntents: immediateModelIntents(value).map(intent => + intent.modelChangeId === settled.modelChangeId ? settled : intent), + }), + auditEvents: [{ + type: 'model_changed', previousModel: invocation.previousModel, + model: settled.model, providerEventId: occurrenceId, + }], + transitionId: `model-invocation:${settled.modelChangeId}:${execution.executionId}:${execution.attemptId}:${occurrenceId}`, + }); + await this.ports.modelChanges.settle(fence, settled.modelChangeId, acknowledgement); + return saved; + } + + private async applyTurnEvent(options: { + fence: GoalSessionFence; current: GoalSessionState; + execution: GoalExecutionIdentity; event: GoalSessionEvent; + }): Promise { + const { fence, current, execution, event } = options; + if (event.type === 'checkpoint') return this.persistCheckpoint(fence, current, execution, event); + if (event.type === 'usage') return this.persistUsage(fence, current, execution, event); + if (event.type === 'model_changed') return this.commitTurnTransition({ + state: current, fence, execution, + update: value => ({ + currentModel: event.model, + pendingModelChange: value.pendingModelChange === event.model ? undefined : value.pendingModelChange, + }), + auditEvents: [event], transitionId: streamAuditTransitionId(fence, execution, event), + }); + if (event.type === 'pause_boundary') return this.commitTurnTransition({ + state: current, fence, execution, + update: value => ({ + status: 'paused', + activeTurn: value.activeTurn ? { ...value.activeTurn, status: 'paused' } : value.activeTurn, + }), + auditEvents: [event], transitionId: streamAuditTransitionId(fence, execution, event), + }); + if (event.type === 'completion') return this.commitTurnCompletion(fence, execution, event); + return current; + } + + private persistUsage( + fence: GoalSessionFence, + state: GoalSessionState, + execution: GoalExecutionIdentity, + event: Extract, + ): Promise { + const accounting = state.usageAccounting ?? { version: 1 as const, lastWatermark: -1, occurrences: [] }; + if (accounting.occurrences.includes(event.occurrenceId) || event.watermark <= accounting.lastWatermark) { + return Promise.resolve(state); + } + if (event.semantics === 'delta' && event.watermark !== accounting.lastWatermark + 1) { + throw new GoalSessionContractError('Provider delta usage skipped a durable watermark', 'USAGE_WATERMARK_GAP'); + } + return this.commitTurnTransition({ + state, fence, execution, + update: value => { + const current = value.usageAccounting ?? { version: 1 as const, lastWatermark: -1, occurrences: [] }; + if (current.occurrences.includes(event.occurrenceId) || event.watermark <= current.lastWatermark) return {}; + return { + usageAccounting: { + version: 1 as const, + lastWatermark: event.watermark, + occurrences: [...current.occurrences, event.occurrenceId].slice(-256), + }, + }; + }, + auditEvents: [event], + transitionId: `usage:${execution.executionId}:${execution.attemptId}:${event.occurrenceId}:${event.watermark}`, + }); + } + + private persistCheckpoint( + fence: GoalSessionFence, + state: GoalSessionState, + execution: GoalExecutionIdentity, + event: Extract, + ): Promise { + if (event.providerSessionId && state.providerSessionId && event.providerSessionId !== state.providerSessionId) { + throw new GoalSessionContractError('Checkpoint attempted to replace the provider session identity', 'PROVIDER_SESSION_CHANGED'); + } + assertCredentialFreeRecoveryMetadata(event.recoveryMetadata, this.adapter.provider); + return this.updateActiveTurnState(fence, execution, value => ({ + ...value, + providerSessionId: event.providerSessionId ?? value.providerSessionId, + recoveryMetadata: sanitizeNewRecoveryMetadata(event.recoveryMetadata, this.adapter.provider), + initializationIntent: event.providerSessionId ? undefined : value.initializationIntent, + currentModel: event.providerSessionId && !value.providerSessionId + ? value.activeTurn?.requestedModel ?? value.currentModel : value.currentModel, + pendingModelChange: event.providerSessionId && !value.providerSessionId + && value.pendingModelChange === value.activeTurn?.requestedModel + ? undefined : value.pendingModelChange, + })); + } + + private async finishTurnIfOwned( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + error: string, + ): Promise { + try { + return await this.commitTurnCompletion(fence, execution, { type: 'completion', outcome: 'failed', error }); + } catch { + return this.requireState(fence); + } + } +} + +function recoverableFirstTurnIdentityLoss( + error: GoalSessionContractError, + capabilities: GoalSessionCore['adapter']['capabilities'], +): boolean { + return error.code === 'FIRST_TURN_ID_NOT_BOUND' && capabilities.nativeSessionId === 'first_turn' + && capabilities.firstTurnIdCrashPolicy === 'retry_deterministically'; +} + +function stopsAtActivePause(event: GoalSessionEvent, pause: 'active_turn' | 'after_turn'): boolean { + return event.type === 'pause_boundary' && pause === 'active_turn'; +} + +function invocationEvidenceOccurrence(event: GoalSessionEvent): string | undefined { + return event.type === 'model_changed' + ? event.providerEventId ?? (event.providerEventOrdinal === undefined ? undefined : `ordinal-${event.providerEventOrdinal}`) + : undefined; +} + +function sameHistoricalModelOccurrence( + intent: GoalModelChangeIntent, + event: Extract, + occurrenceId: string, +): boolean { + const previous = intent.previousInvocationEvidence; + return previous?.modelChangeId === intent.modelChangeId && previous.generation === intent.generation + && previous.occurrenceId === occurrenceId && previous.requestedModel === intent.model + && previous.effectiveModel === event.model; +} + +function modelInvocationEvidence( + intent: GoalModelChangeIntent, + execution: GoalExecutionIdentity, + occurrenceId: string, + effectiveModel: string, +) { + return { + ...execution, modelChangeId: intent.modelChangeId, generation: intent.generation!, occurrenceId, + requestedModel: intent.model, effectiveModel, acceptedAt: new Date().toISOString(), + }; +} + +function consumesNextTurnModelEvidence( + state: GoalSessionState, + event: GoalSessionEvent, + execution: GoalExecutionIdentity, + capability: 'next_safe_boundary' | 'next_turn', +): boolean { + if (event.type !== 'model_changed' || capability !== 'next_turn' || !state.activeTurn?.modelChange) return false; + const invocation = state.activeTurn.modelChange; + const evidence = immediateModelIntents(state).find(intent => + intent.modelChangeId === invocation.modelChangeId)?.invocationEvidence; + if (!evidence) return true; + const occurrenceId = invocationEvidenceOccurrence(event); + return evidence.executionId === execution.executionId && evidence.attemptId === execution.attemptId + && evidence.occurrenceId === occurrenceId && evidence.effectiveModel === event.model; +} + +function assertExactNextTurnModelEvidence(state: GoalSessionState, execution: GoalExecutionIdentity): void { + const invocation = state.activeTurn?.modelChange; + if (!invocation) return; + const intent = immediateModelIntents(state).find(candidate => + candidate.modelChangeId === invocation.modelChangeId && candidate.generation === invocation.generation); + const evidence = intent?.invocationEvidence; + if (!intent || intent.phase !== 'committed' || intent.acknowledgement?.appliesAt !== 'next_turn' + || intent.acknowledgement.outcome !== 'acknowledged' + || evidence?.executionId !== execution.executionId || evidence.attemptId !== execution.attemptId + || evidence.modelChangeId !== invocation.modelChangeId || evidence.generation !== invocation.generation + || evidence.requestedModel !== intent.model || evidence.effectiveModel !== intent.model) { + throw new GoalSessionContractError( + 'Turn completion lacks exact authoritative next-turn model evidence', 'MODEL_EVIDENCE_MISSING', + ); + } +} + +function unchangedStreamProgress(state: GoalSessionState, completed: boolean): TurnStreamProgress { + return { state, completed, reachedPause: false, stop: false }; +} diff --git a/packages/core/src/agents/goalSession/InMemoryGoalSessionPorts.ts b/packages/core/src/agents/goalSession/InMemoryGoalSessionPorts.ts new file mode 100644 index 000000000..c1fa05000 --- /dev/null +++ b/packages/core/src/agents/goalSession/InMemoryGoalSessionPorts.ts @@ -0,0 +1,417 @@ +import type { + DurableCorrectiveMessage, + GoalContainerInspection, + GoalEventAppendResult, + GoalExecutionIdentity, + GoalRepositoryIdentity, + GoalRepositoryInspection, + GoalProviderFirstEffectPort, + GoalProviderEffectStage, + GoalProviderOperationFence, + GoalStartedProviderEffect, + GoalSessionControlFence, + GoalSessionControlTransition, + GoalSessionEvent, + GoalSessionEventSink, + GoalSessionFence, + GoalSessionIdentity, + GoalSessionMessagePort, + GoalSessionRecoveryPort, + GoalSessionRuntimePorts, + GoalSessionState, + GoalSessionStatePort, + GoalSessionTerminalPort, + GoalSessionTransitionPort, + GoalTerminalCommit, + PersistedGoalSessionEvent, +} from './contract.js'; +import { InMemoryModelChangeHistory } from './InMemoryModelChangeHistory.js'; +import { + matchesLiveMessageFence, matchesTransitionLiveFence, terminalCommitKey, transitionCommitKey, +} from './inMemoryGoalSessionFences.js'; +import { sanitizeGoalSessionEvent } from './securityBoundary.js'; +import { assertProviderFirstEffectState } from './providerFirstEffect.js'; +import { assertStartedProviderEffect } from './providerEffectProtocol.js'; +import { assertGoalProviderEffectStage } from './providerOperationBoundary.js'; +import { GoalSessionScopeError } from './errors.js'; +export { GoalSessionScopeError } from './errors.js'; + +function clone(value: T): T { + return structuredClone(value); +} + +function keyOf(identity: GoalSessionIdentity): string { + return `${identity.goalId}\0${identity.sessionId}`; +} + +/** + * Deterministic in-memory durable-port fake used by contract tests and by + * embedders that want the runtime without a database. All state/event/message + * mutations are synchronous inside each async method, which gives the same + * atomic fence semantics expected from a database transaction. + * + * This is explicitly NOT a production durability fallback: everything lives in + * process memory and is lost on restart, so it must never back a real goal that + * needs to survive a worker/daemon crash. Production deployments provide a + * transactional {@link GoalSessionRuntimePorts} implementation instead. + */ +export class InMemoryGoalSessionPorts implements + GoalSessionStatePort, + GoalSessionEventSink, + GoalSessionMessagePort, + GoalSessionRecoveryPort, + GoalProviderFirstEffectPort, + GoalSessionTerminalPort, + GoalSessionTransitionPort { + /** Marks this implementation as an ephemeral test/embedding double, never durable storage. */ + readonly isEphemeralTestDouble = true; + + private readonly states = new Map(); + private readonly sessionOwners = new Map(); + private readonly events = new Map(); + private readonly messages = new Map(); + private readonly containerInspections = new Map(); + private readonly repositoryInspections = new Map(); + private readonly terminalCommits = new Set(); + private readonly transitionCommits = new Set(); + private readonly modelChangeHistory = new InMemoryModelChangeHistory(); + private terminalFault: 'before_commit' | 'before_commit_always' | 'after_commit' | undefined; + private transitionFault: 'before_commit' | 'after_commit' | undefined; + + asRuntimePorts(): GoalSessionRuntimePorts { + return { + state: this, transitions: this, events: this, terminal: this, + messages: this, recovery: this, modelChanges: this.modelChangeHistory, + providerFirstEffects: this, + }; + } + + async start( + fence: GoalProviderOperationFence, + _stage: GoalProviderEffectStage, + effect: () => GoalStartedProviderEffect, + rebuild: (value: T) => R, + ): Promise { + assertGoalProviderEffectStage(_stage); + const state = this.states.get(keyOf(fence)); + assertProviderFirstEffectState(state ? clone(state) : null, fence); + const started = effect(); + assertStartedProviderEffect(started); + return rebuild(await started.completion); + } + + async load(identity: GoalSessionIdentity): Promise { + this.assertGoalScope(identity); + const state = this.states.get(keyOf(identity)); + return state ? clone(state) : null; + } + + async create(state: Omit): Promise { + this.assertGoalScope(state); + const key = keyOf(state); + if (this.states.has(key)) return null; + const saved = { ...clone(state), version: 1 }; + this.sessionOwners.set(state.sessionId, state.goalId); + this.states.set(key, saved); + return clone(saved); + } + + async compareAndSet( + expected: GoalSessionState, + next: Omit, + ): Promise { + this.assertGoalScope(expected); + this.assertGoalScope(next); + if (expected.goalId !== next.goalId || expected.sessionId !== next.sessionId) { + throw new GoalSessionScopeError('A state update cannot move a session to another goal or identity'); + } + const key = keyOf(expected); + const current = this.states.get(key); + if (!current || current.version !== expected.version) return null; + const saved = { ...clone(next), version: current.version + 1 }; + this.states.set(key, saved); + return clone(saved); + } + + async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ): Promise { + if (!('scope' in operation)) return this.commitTransition(expected, next, operation); + return this.commitTerminal(expected, next, operation); + } + + private commitTerminal( + expected: GoalSessionState, + next: Omit, + completion: GoalTerminalCommit, + ): GoalSessionState | null { + this.assertGoalScope(expected); + this.assertGoalScope(next); + if (expected.goalId !== next.goalId || expected.sessionId !== next.sessionId) { + throw new GoalSessionScopeError('A terminal transaction cannot move a session to another goal or identity'); + } + const key = keyOf(expected); + const current = this.states.get(key); + const commitKey = terminalCommitKey(completion); + const turnId = completion.scope === 'turn' + ? completion.fence.turnId + : `#control-e${completion.fence.controllerEpoch}`; + if (this.terminalCommits.has(commitKey)) return current ? clone(current) : null; + if (!current || current.version !== expected.version + || current.controllerEpoch !== completion.fence.controllerEpoch) return null; + if (completion.scope === 'turn' + && (current.activeTurn?.turnId !== completion.fence.turnId + || current.activeTurn.executionId !== completion.execution.executionId + || current.activeTurn.attemptId !== completion.execution.attemptId)) return null; + if (this.terminalFault === 'before_commit' || this.terminalFault === 'before_commit_always') { + if (this.terminalFault === 'before_commit') this.terminalFault = undefined; + throw new Error('Injected crash before terminal transaction commit'); + } + const saved = { ...clone(next), version: current.version + 1 }; + this.states.set(key, saved); + for (const event of completion.auditEvents ?? []) { + this.record(key, { turnId, fence: completion.fence, execution: completion.execution, event }); + } + this.record(key, { turnId, fence: completion.fence, execution: completion.execution, event: completion.event }); + this.terminalCommits.add(commitKey); + if (this.terminalFault === 'after_commit') { + this.terminalFault = undefined; + throw new Error('Injected crash after terminal transaction commit'); + } + return clone(saved); + } + + /** Test-only crash injection around the atomic terminal transaction. */ + setTerminalFault(fault: 'before_commit' | 'before_commit_always' | 'after_commit' | undefined): void { + this.terminalFault = fault; + } + + /** Test-only crash injection around an atomic nonterminal state/audit transaction. */ + setTransitionFault(fault: 'before_commit' | 'after_commit' | undefined): void { + this.transitionFault = fault; + } + + async append( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + const scopeError = this.scopeRejection(fence); + if (scopeError) return scopeError; + const key = keyOf(fence); + const state = this.states.get(key); + if (!state || state.controllerEpoch !== fence.controllerEpoch) { + return { accepted: false, reason: 'stale_fence' }; + } + if (state.providerBarrierIntent?.phase === 'pending' + || state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed') { + return { accepted: false, reason: 'turn_not_active' }; + } + if (state.activeTurn?.turnId !== fence.turnId + || state.activeTurn.executionId !== execution.executionId + || state.activeTurn.attemptId !== execution.attemptId + || state.activeTurn.status === 'completed' + || state.activeTurn.status === 'cancelled' + || state.activeTurn.status === 'failed') { + return { accepted: false, reason: 'turn_not_active' }; + } + return { accepted: true, persisted: this.record(key, { turnId: fence.turnId, fence, execution, event }) }; + } + + async appendControl( + fence: GoalSessionControlFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + const scopeError = this.scopeRejection(fence); + if (scopeError) return scopeError; + const key = keyOf(fence); + const state = this.states.get(key); + if (!state || state.controllerEpoch !== fence.controllerEpoch) { + return { accepted: false, reason: 'stale_fence' }; + } + if (state.providerBarrierIntent?.phase === 'pending' + || state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed') { + return { accepted: false, reason: 'stale_fence' }; + } + // Control events are session/epoch fenced only, so they remain auditable + // in idle state and are never attributed to a specific (possibly + // completed) turn. + return { accepted: true, persisted: this.record(key, { turnId: `#control-e${fence.controllerEpoch}`, fence, execution, event }) }; + } + + private scopeRejection(identity: GoalSessionIdentity): { accepted: false; reason: 'wrong_goal' } | null { + try { this.assertGoalScope(identity); return null; } + catch (error) { + if (error instanceof GoalSessionScopeError) return { accepted: false, reason: 'wrong_goal' }; + throw error; + } + } + + private record( + key: string, + entry: { turnId: string; fence: GoalSessionControlFence; execution: GoalExecutionIdentity; event: GoalSessionEvent }, + ): PersistedGoalSessionEvent { + const log = this.events.get(key) ?? []; + const persisted: PersistedGoalSessionEvent = { + goalId: entry.fence.goalId, + sessionId: entry.fence.sessionId, + controllerEpoch: entry.fence.controllerEpoch, + turnId: entry.turnId, + executionId: entry.execution.executionId, + attemptId: entry.execution.attemptId, + sequence: (log.at(-1)?.sequence ?? 0) + 1, + recordedAt: new Date().toISOString(), + event: clone(sanitizeGoalSessionEvent(entry.event)), + }; + log.push(persisted); + this.events.set(key, log); + return clone(persisted); + } + + async replay(identity: GoalSessionIdentity, afterSequence = 0): Promise { + this.assertGoalScope(identity); + return clone((this.events.get(keyOf(identity)) ?? []).filter(event => event.sequence > afterSequence)); + } + + async listPending(identity: GoalSessionIdentity): Promise { + this.assertGoalScope(identity); + return clone((this.messages.get(keyOf(identity)) ?? []).filter(message => !message.acknowledgedAt)); + } + + async acknowledge( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + ): Promise<'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found'> { + this.assertGoalScope(fence); + const state = this.states.get(keyOf(fence)); + if (!state || state.controllerEpoch !== fence.controllerEpoch + || state.providerBarrierIntent?.phase === 'pending' + || state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed' + || state.activeTurn?.turnId !== fence.turnId + || state.activeTurn.executionId !== execution.executionId + || state.activeTurn.attemptId !== execution.attemptId + || state.activeTurn.status === 'completed' + || state.activeTurn.status === 'cancelled' + || state.activeTurn.status === 'failed') { + return 'stale_fence'; + } + const records = this.messages.get(keyOf(fence)) ?? []; + const record = records.find(message => message.messageId === messageId); + if (!record) return 'not_found'; + if (record.acknowledgedAt) return 'already_acknowledged'; + record.acknowledgedAt = new Date().toISOString(); + return 'acknowledged'; + } + + async acknowledgeWithEvent( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + ): Promise<'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found'> { + this.assertGoalScope(fence); + const state = this.states.get(keyOf(fence)); + if (!matchesLiveMessageFence(state, fence, execution)) return 'stale_fence'; + const record = (this.messages.get(keyOf(fence)) ?? []).find(message => message.messageId === messageId); + if (!record) return 'not_found'; + if (record.acknowledgedAt) return 'already_acknowledged'; + record.acknowledgedAt = new Date().toISOString(); + this.record(keyOf(fence), { + turnId: fence.turnId, fence, execution, + event: { type: 'message_acknowledged', messageId }, + }); + return 'acknowledged'; + } + + /** Test/application helper representing the persistence side of the message port. */ + enqueueMessage(message: Omit & Partial>): DurableCorrectiveMessage { + this.assertGoalScope(message); + const key = keyOf(message); + const records = this.messages.get(key) ?? []; + if (records.some(record => record.messageId === message.messageId)) { + throw new Error(`Corrective message "${message.messageId}" already exists`); + } + const record: DurableCorrectiveMessage = { + ...message, + sequence: message.sequence ?? (records.at(-1)?.sequence ?? 0) + 1, + createdAt: message.createdAt ?? new Date().toISOString(), + }; + records.push(record); + records.sort((a, b) => a.sequence - b.sequence); + this.messages.set(key, records); + return clone(record); + } + + setContainerInspection(identity: GoalSessionIdentity, inspection: GoalContainerInspection): void { + this.assertGoalScope(identity); + this.containerInspections.set(keyOf(identity), clone(inspection)); + } + + setRepositoryInspection(repository: GoalRepositoryIdentity, inspection: GoalRepositoryInspection): void { + this.repositoryInspections.set(repository.worktreePath, clone(inspection)); + } + + async inspectContainer(identity: GoalSessionIdentity): Promise { + this.assertGoalScope(identity); + return clone(this.containerInspections.get(keyOf(identity)) ?? { + status: 'missing', + reason: 'No goal-scoped container was found', + }); + } + + async inspectRepository(repository: GoalRepositoryIdentity): Promise { + return clone(this.repositoryInspections.get(repository.worktreePath) ?? { + ...repository, + exists: false, + reason: 'The goal worktree was not found', + }); + } + + private assertGoalScope(identity: GoalSessionIdentity): void { + const owner = this.sessionOwners.get(identity.sessionId); + if (owner !== undefined && owner !== identity.goalId) throw new GoalSessionScopeError(); + } + + private commitTransition( + expected: GoalSessionState, + next: Omit, + transition: GoalSessionControlTransition, + ): GoalSessionState | null { + this.assertGoalScope(expected); + this.assertGoalScope(next); + if (expected.goalId !== next.goalId || expected.sessionId !== next.sessionId) { + throw new GoalSessionScopeError('A state/audit transaction cannot move a session to another goal or identity'); + } + const key = keyOf(expected); + const current = this.states.get(key); + const commitKey = transitionCommitKey(transition); + if (!matchesTransitionLiveFence(current, transition)) return null; + if (this.transitionCommits.has(commitKey)) return clone(current); + if (current.version !== expected.version) return null; + if (this.transitionFault === 'before_commit') { + this.transitionFault = undefined; + throw new Error('Injected crash before state/audit transaction commit'); + } + const saved = { ...clone(next), version: current.version + 1 }; + this.states.set(key, saved); + for (const event of transition.auditEvents) { + this.record(key, { + turnId: transition.turnScoped === true && 'turnId' in transition.fence + ? transition.fence.turnId + : `#control-e${transition.fence.controllerEpoch}`, + fence: transition.fence, + execution: transition.execution, + event, + }); + } + this.transitionCommits.add(commitKey); + if (this.transitionFault === 'after_commit') { + this.transitionFault = undefined; + throw new Error('Injected crash after state/audit transaction commit'); + } + return clone(saved); + } +} diff --git a/packages/core/src/agents/goalSession/InMemoryModelChangeHistory.ts b/packages/core/src/agents/goalSession/InMemoryModelChangeHistory.ts new file mode 100644 index 000000000..8f0efa8b4 --- /dev/null +++ b/packages/core/src/agents/goalSession/InMemoryModelChangeHistory.ts @@ -0,0 +1,39 @@ +import type { + GoalModelChangeAcknowledgement, GoalModelChangeHistoryPort, + GoalModelChangeHistoryRecord, GoalSessionIdentity, +} from './contract.js'; + +type SequencedRecord = GoalModelChangeHistoryRecord & { sequence: number }; + +export class InMemoryModelChangeHistory implements GoalModelChangeHistoryPort { + private readonly records = new Map(); + + async claim(identity: GoalSessionIdentity, operationId: string, model: string): Promise { + const key = scope(identity); + const records = this.records.get(key) ?? []; + const existing = records.find(record => record.operationId === operationId); + if (existing) return structuredClone(existing); + const created = { operationId, model, status: 'pending' as const, sequence: (records.at(-1)?.sequence ?? 0) + 1 }; + records.push(created); + this.records.set(key, records); + return structuredClone(created); + } + + async settle( + identity: GoalSessionIdentity, + operationId: string, + acknowledgement: GoalModelChangeAcknowledgement, + ): Promise { + const records = this.records.get(scope(identity)) ?? []; + const record = records.find(value => value.operationId === operationId); + if (!record) return; + record.status = 'settled'; + record.acknowledgement = structuredClone(acknowledgement); + const settled = records.filter(value => value.status === 'settled'); + for (const retired of settled.slice(0, Math.max(0, settled.length - 64))) retired.status = 'retired'; + } +} + +function scope(identity: GoalSessionIdentity): string { + return `${identity.goalId}\0${identity.sessionId}`; +} diff --git a/packages/core/src/agents/goalSession/SqliteGoalSessionControlDomain.ts b/packages/core/src/agents/goalSession/SqliteGoalSessionControlDomain.ts new file mode 100644 index 000000000..0bcf33eff --- /dev/null +++ b/packages/core/src/agents/goalSession/SqliteGoalSessionControlDomain.ts @@ -0,0 +1,543 @@ +import { createHash, randomUUID } from 'node:crypto'; +import Database from 'better-sqlite3'; +import type { + DurableCorrectiveMessage, GoalEventAppendResult, GoalExecutionIdentity, + GoalModelChangeAcknowledgement, GoalModelChangeHistoryRecord, GoalProviderEffectStage, + GoalProviderOperationFence, GoalSessionControlFence, GoalSessionControlTransition, + GoalSessionEvent, GoalSessionFence, GoalSessionIdentity, GoalSessionState, + GoalStartedProviderEffect, GoalTerminalCommit, PersistedGoalSessionEvent, +} from './contract.js'; +import type { + GoalProviderEffectClaimResult, GoalSessionAuthoritativeTransactionDomain, +} from './AuthoritativeGoalSessionRuntimePorts.js'; +import { AuthoritativeGoalSessionRuntimePorts } from './AuthoritativeGoalSessionRuntimePorts.js'; +import type { GoalSessionRecoveryPort, GoalSessionRuntimePorts } from './runtimePorts.js'; +import { GoalSessionContractError, GoalSessionScopeError } from './errors.js'; +import { assertStartedProviderEffect } from './providerEffectProtocol.js'; +import { assertProviderFirstEffectState } from './providerFirstEffect.js'; +import { assertGoalProviderEffectStage, assertGoalProviderOperationFence } from './providerOperationBoundary.js'; +import { sanitizeGoalSessionEvent } from './securityBoundary.js'; +import { sqliteGoalScope, sqliteTerminalKey, sqliteTransitionKey } from './sqliteGoalSessionKeys.js'; +import { assertSqliteGoalControlSchema, replayableProviderOutcomeJson } from './sqliteGoalSessionSchema.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +type EffectRow = { kind: string; status: string; claim_token: string; outcome_json: string | null }; + +/** Production SQLite adapter over an injected, already-migrated control database. */ +export class SqliteGoalSessionControlDomain implements GoalSessionAuthoritativeTransactionDomain { + readonly state = this; readonly transitions = this; readonly events = this; readonly terminal = this; readonly messages = this; readonly modelChanges = this; readonly providerEffects = this; + + constructor(private readonly database: Database.Database) { + database.pragma('foreign_keys = ON'); database.pragma('busy_timeout = 30000'); + assertSqliteGoalControlSchema(database); + } + + async load(identity: GoalSessionIdentity): Promise { + this.assertOwner(identity); + return this.readState(identity); + } + + async create(state: Omit): Promise { + const saved = { ...structuredClone(state), version: 1 }; + return this.immediate(() => { + this.bindOwnerForCreate(state); + this.assertOwner(state); + const result = this.database.prepare( + `INSERT OR IGNORE INTO goal_session_runtime_state + (session_id, goal_id, scope, payload_json) VALUES (?, ?, ?, ?)`, + ).run(state.sessionId, state.goalId, sqliteGoalScope(state), JSON.stringify(saved)); + if (result.changes !== 1) return null; + this.syncProviderSession(saved); + return saved; + }); + } + + async compareAndSet( + expected: GoalSessionState, + next: Omit, + ): Promise { + return this.immediate(() => this.writeComparedState(expected, next)); + } + + async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ): Promise { + return this.immediate(() => { + this.assertOwner(expected); + return 'scope' in operation + ? this.commitTerminal(expected, next, operation) + : this.commitTransition(expected, next, operation); + }); + } + + async append( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + return this.immediate(() => { + this.assertOwner(fence); + return matchesTurn(this.readState(fence), fence, execution) + ? { accepted: true, persisted: this.record(fence, fence.turnId, execution, event) } + : { accepted: false, reason: 'turn_not_active' }; + }); + } + + async appendControl( + fence: GoalSessionControlFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + return this.immediate(() => { + this.assertOwner(fence); + const state = this.readState(fence); + if (!matchesControl(state, fence)) return { accepted: false as const, reason: 'stale_fence' as const }; + return { accepted: true as const, persisted: this.record(fence, `#control-e${fence.controllerEpoch}`, execution, event) }; + }); + } + + async replay(identity: GoalSessionIdentity, afterSequence = 0): Promise { + this.assertOwner(identity); + const rows = this.database.prepare( + `SELECT payload_json FROM goal_events + WHERE goal_id = ? AND sequence > ? AND kind = 'domain' + AND event_type LIKE 'goal_session.%' ORDER BY sequence`, + ).all(identity.goalId, afterSequence) as Array<{ payload_json: string | null }>; + return rows.flatMap(row => { + if (!row.payload_json) return []; + let event: unknown; + try { event = JSON.parse(row.payload_json); } + catch { return []; } + return isPersistedRuntimeEvent(event, identity) ? [event] : []; + }); + } + + async listPending(identity: GoalSessionIdentity): Promise { + this.assertOwner(identity); + const rows = this.database.prepare( + `SELECT message_id, sequence, body, created_at, acknowledged_at FROM goal_messages + WHERE goal_id = ? AND state IN ('queued', 'delivering', 'delivered') ORDER BY queue_ordinal`, + ).all(identity.goalId) as Array<{ + message_id: string; sequence: number; body: string; created_at: string; acknowledged_at: string | null; + }>; + return rows.map(row => ({ + ...identity, messageId: row.message_id, sequence: row.sequence, body: row.body, + createdAt: row.created_at, acknowledgedAt: row.acknowledged_at ?? undefined, + })); + } + + async acknowledgeWithEvent( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + ): Promise<'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found'> { + return this.immediate(() => { + this.assertOwner(fence); + if (!matchesTurn(this.readState(fence), fence, execution)) return 'stale_fence'; + const row = this.database.prepare(`SELECT state, delivered_at FROM goal_messages + WHERE goal_id = ? AND message_id = ?`).get(fence.goalId, messageId) as { + state: string; delivered_at: string | null; + } | undefined; + if (!row) return 'not_found'; + if (row.state === 'acknowledged') return 'already_acknowledged'; + if (!['queued', 'delivering', 'delivered'].includes(row.state)) return 'not_found'; + const acknowledgedAt = new Date().toISOString(); + if (row.state === 'queued') this.database.prepare(`UPDATE goal_messages SET state = 'delivering', + claimed_by = ?, claimed_turn_id = ?, claimed_lease_generation = ?, + delivery_key = ?, delivery_attempts = delivery_attempts + 1 + WHERE goal_id = ? AND message_id = ? AND state = 'queued'`) + .run(fence.sessionId, fence.turnId, fence.controllerEpoch, + boundedEventKey(['message-delivery', fence.sessionId, fence.turnId, messageId]), fence.goalId, messageId); + if (row.state !== 'delivered') this.database.prepare(`UPDATE goal_messages SET state = 'delivered', delivered_at = ? + WHERE goal_id = ? AND message_id = ? AND state = 'delivering'`) + .run(acknowledgedAt, fence.goalId, messageId); + const result = this.database.prepare(`UPDATE goal_messages SET state = 'acknowledged', acknowledged_at = ? + WHERE goal_id = ? AND message_id = ? AND state = 'delivered' AND delivered_at IS NOT NULL`) + .run(acknowledgedAt, fence.goalId, messageId); + if (result.changes !== 1) return 'stale_fence'; + this.record(fence, fence.turnId, execution, { type: 'message_acknowledged', messageId }); + return 'acknowledged'; + }); + } + + async claim(identity: GoalSessionIdentity, operationId: string, model: string): Promise { + return this.immediate(() => { + this.assertOwner(identity); + const existing = this.readModelChange(identity, operationId); + if (existing) return existing; + const row = this.database.prepare(` + INSERT INTO goal_session_runtime_model_sequences + (session_id, goal_id, scope, next_sequence) VALUES (?, ?, ?, 2) + ON CONFLICT(scope) DO UPDATE SET next_sequence = next_sequence + 1 + RETURNING next_sequence - 1 AS sequence + `).get(identity.sessionId, identity.goalId, sqliteGoalScope(identity)) as { sequence: number }; + this.database.prepare(`INSERT INTO goal_session_runtime_model_changes + (session_id, goal_id, scope, operation_id, sequence, model, status) + VALUES (?, ?, ?, ?, ?, ?, 'pending')`) + .run(identity.sessionId, identity.goalId, sqliteGoalScope(identity), operationId, row.sequence, model); + return { operationId, model, sequence: row.sequence, status: 'pending' }; + }); + } + + async settle( + identity: GoalSessionIdentity, + operationId: string, + acknowledgement: GoalModelChangeAcknowledgement, + ): Promise { + this.immediate(() => { + this.assertOwner(identity); + this.database.prepare(`UPDATE goal_session_runtime_model_changes + SET status = 'settled', acknowledgement_json = ? WHERE scope = ? AND operation_id = ?`) + .run(JSON.stringify(acknowledgement), sqliteGoalScope(identity), operationId); + this.database.prepare(`UPDATE goal_session_runtime_model_changes SET status = 'retired', acknowledgement_json = NULL + WHERE scope = ? AND status = 'settled' AND operation_id NOT IN ( + SELECT operation_id FROM goal_session_runtime_model_changes + WHERE scope = ? AND status = 'settled' ORDER BY sequence DESC LIMIT 64 + )`).run(sqliteGoalScope(identity), sqliteGoalScope(identity)); + }); + } + + async claimProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + ): Promise { + assertGoalProviderOperationFence(fence); assertGoalProviderEffectStage(stage); + return this.immediate(() => { + this.assertOwner(fence); + assertProviderFirstEffectState(this.readState(fence), fence); + const current = this.readEffect(fence, stage); + if (current && current.kind !== fence.kind) { + throw new GoalSessionContractError('Provider effect kind conflicts with its durable claim', 'PROVIDER_EFFECT_IN_DOUBT'); + } + if (current?.status === 'settled') { + return { status: 'settled', outcome: JSON.parse(current.outcome_json ?? 'null') }; + } + if (current && (current.status === 'started' || current.status === 'poisoned')) { + return { status: 'terminal_in_doubt' }; + } + const token = randomUUID(); + if (current) { + const result = this.database.prepare(`UPDATE goal_session_runtime_provider_effects + SET status = 'recoverable', claim_token = ?, updated_at = ? + WHERE scope = ? AND operation_id = ? AND stage = ? + AND claim_token = ? AND status IN ('unstarted', 'recoverable')`) + .run(token, new Date().toISOString(), sqliteGoalScope(fence), fence.operationId, stage, current.claim_token); + if (result.changes !== 1) throw effectCasFailure(); + return { status: 'recoverable', token }; + } + this.database.prepare(`INSERT INTO goal_session_runtime_provider_effects + (session_id, goal_id, scope, operation_id, kind, stage, status, claim_token, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'unstarted', ?, ?)`) + .run(fence.sessionId, fence.goalId, sqliteGoalScope(fence), fence.operationId, + fence.kind, stage, token, new Date().toISOString()); + return { status: 'claimed', token }; + }); + } + + async runClaimedProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + token: string, + effect: () => GoalStartedProviderEffect, + ): Promise> { + assertGoalProviderOperationFence(fence); assertGoalProviderEffectStage(stage); + if (!isSafeIdentifier(token)) throw effectCasFailure(); + // Persist provider entry before callback execution. A crash after this + // CAS is conservatively live/unknown and can never re-enter callback. + this.immediate(() => { + this.assertOwner(fence); + assertProviderFirstEffectState(this.readState(fence), fence); + const result = this.database.prepare(`UPDATE goal_session_runtime_provider_effects + SET status = 'started', updated_at = ? WHERE scope = ? AND operation_id = ? + AND stage = ? AND claim_token = ? AND status IN ('unstarted', 'recoverable')`) + .run(new Date().toISOString(), sqliteGoalScope(fence), fence.operationId, stage, token); + if (result.changes !== 1) throw effectCasFailure(); + }); + // Re-lock and revalidate the authoritative state immediately around the + // synchronous primitive start. Invalidations on another connection are + // serialized before or after this callback, never through it. + return this.immediate(() => { + this.assertOwner(fence); + assertProviderFirstEffectState(this.readState(fence), fence); + const claim = this.readEffect(fence, stage); + if (!claim || claim.kind !== fence.kind || claim.status !== 'started' + || claim.claim_token !== token) throw effectCasFailure(); + const started = effect(); + assertStartedProviderEffect(started); + return started; + }); + } + + async settleProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + token: string, + outcome: unknown, + ): Promise { + assertGoalProviderOperationFence(fence); assertGoalProviderEffectStage(stage); + if (!isSafeIdentifier(token)) throw effectCasFailure(); + const outcomeJson = stage === 'container_spawn' ? null : replayableProviderOutcomeJson(outcome); + this.immediate(() => { + this.assertOwner(fence); + const result = this.database.prepare(`UPDATE goal_session_runtime_provider_effects + SET status = ?, outcome_json = ?, updated_at = ? WHERE scope = ? AND operation_id = ? + AND stage = ? AND status = 'started' AND claim_token = ?`) + .run(stage === 'container_spawn' ? 'poisoned' : 'settled', outcomeJson, + new Date().toISOString(), sqliteGoalScope(fence), fence.operationId, stage, token); + if (result.changes !== 1) throw effectCasFailure(); + }); + } + + async poisonProviderEffect(fence: GoalProviderOperationFence, stage: GoalProviderEffectStage, token: string): Promise { + assertGoalProviderOperationFence(fence); assertGoalProviderEffectStage(stage); + if (!isSafeIdentifier(token)) throw effectCasFailure(); + this.immediate(() => { + this.assertOwner(fence); + this.database.prepare(`UPDATE goal_session_runtime_provider_effects SET status = 'poisoned', updated_at = ? + WHERE scope = ? AND operation_id = ? AND stage = ? AND claim_token = ? AND status != 'settled'`) + .run(new Date().toISOString(), sqliteGoalScope(fence), fence.operationId, stage, token); + }); + } + + private commitTransition( + expected: GoalSessionState, + next: Omit, + transition: GoalSessionControlTransition, + ): GoalSessionState | null { + const current = this.readState(expected); + if (!matchesTransition(current, transition)) return null; + const identity = sqliteTransitionKey(transition); + if (this.hasCommit('transition', identity)) return current; + if (current.version !== expected.version) return null; + const saved = this.writeComparedState(expected, next); + if (!saved) return null; + for (const event of transition.auditEvents) this.record( + transition.fence, + transition.turnScoped === true && 'turnId' in transition.fence + ? transition.fence.turnId : `#control-e${transition.fence.controllerEpoch}`, + transition.execution, event, + ); + this.addCommit(transition.fence, 'transition', identity); + return saved; + } + + private commitTerminal( + expected: GoalSessionState, + next: Omit, + completion: GoalTerminalCommit, + ): GoalSessionState | null { + const current = this.readState(expected); + const identity = sqliteTerminalKey(completion); + if (!current || current.version !== expected.version || current.controllerEpoch !== completion.fence.controllerEpoch + || completion.scope === 'turn' && !matchesTurn(current, completion.fence, completion.execution)) return null; + if (this.hasCommit('terminal', identity)) return current; + const saved = this.writeComparedState(expected, next); + if (!saved) return null; + const turnId = completion.scope === 'turn' + ? completion.fence.turnId : `#control-e${completion.fence.controllerEpoch}`; + for (const event of completion.auditEvents) this.record(completion.fence, turnId, completion.execution, event); + this.record(completion.fence, turnId, completion.execution, completion.event); + this.addCommit(completion.fence, 'terminal', identity); + return saved; + } + + private writeComparedState( + expected: GoalSessionState, + next: Omit, + ): GoalSessionState | null { + this.assertOwner(expected); + const current = this.readState(expected); + if (current?.version !== expected.version) return null; + const saved = { ...structuredClone(next), version: expected.version + 1 }; + const result = this.database.prepare(`UPDATE goal_session_runtime_state SET payload_json = ? + WHERE scope = ? AND payload_json = ?`) + .run(JSON.stringify(saved), sqliteGoalScope(expected), JSON.stringify(current)); + if (result.changes !== 1) return null; + this.syncProviderSession(saved); + return saved; + } + + private syncProviderSession(state: GoalSessionState): void { + const metadata = state.recoveryMetadata === undefined ? null : replayableProviderOutcomeJson(state.recoveryMetadata); + const result = this.database.prepare(`UPDATE goal_provider_sessions SET + provider_thread_id = ?, recovery_metadata_json = ?, + effective_model = COALESCE(?, effective_model), lease_generation = ?, + current_turn_id = ?, current_execution_id = ?, current_attempt_id = ?, updated_at = ? + WHERE session_id = ? AND goal_id = ?`) + .run(state.providerSessionId ?? null, metadata, state.currentModel ?? null, state.controllerEpoch, + state.activeTurn?.turnId ?? null, state.activeTurn?.executionId ?? null, + state.activeTurn?.attemptId ?? null, new Date().toISOString(), state.sessionId, state.goalId); + if (result.changes !== 1) throw new GoalSessionScopeError(); + } + + private record( + fence: GoalSessionControlFence, + turnId: string, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): PersistedGoalSessionEvent { + const sequence = this.allocateEventSequence(fence.goalId); + const persisted: PersistedGoalSessionEvent = { + ...fence, turnId, ...execution, sequence, + recordedAt: new Date().toISOString(), event: structuredClone(sanitizeGoalSessionEvent(event)), + }; + const payloadJson = replayableProviderOutcomeJson(persisted); + if (Buffer.byteLength(payloadJson, 'utf8') > 256 * 1024) throw new GoalSessionContractError( + 'Goal runtime event exceeds the authoritative payload bound', 'UNSAFE_PROVIDER_VALUE', + ); + this.database.prepare(`INSERT INTO goal_events + (goal_id, sequence, kind, event_type, payload_json, idempotency_key, lease_epoch, created_at, + schema_version, source_session_id, source_turn_id, source_execution_id, source_attempt_id, + lease_generation, payload_bytes) + VALUES (?, ?, 'domain', ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?)`) + .run(fence.goalId, persisted.sequence, `goal_session.${persisted.event.type}`, payloadJson, + boundedEventKey(['goal-session', fence.sessionId, String(sequence)]), + fence.controllerEpoch, persisted.recordedAt, fence.sessionId, turnId, + execution.executionId, execution.attemptId, fence.controllerEpoch, + Buffer.byteLength(payloadJson, 'utf8')); + this.database.prepare(`UPDATE goal_event_state SET projection_sequence = ?, updated_at = ? + WHERE goal_id = ? AND projection_sequence < ?`) + .run(sequence, persisted.recordedAt, fence.goalId, sequence); + return persisted; + } + + private allocateEventSequence(goalId: string): number { + this.database.prepare(`INSERT OR IGNORE INTO goal_event_state + (goal_id, high_watermark, min_retained_sequence, projection_sequence, checkpoint_sequence, updated_at) + SELECT goal_id, COALESCE((SELECT MAX(sequence) FROM goal_events WHERE goal_id = ?), 0), 1, + COALESCE((SELECT MAX(sequence) FROM goal_events WHERE goal_id = ?), 0), 0, ? + FROM goals WHERE goal_id = ?`) + .run(goalId, goalId, new Date().toISOString(), goalId); + const row = this.database.prepare(`UPDATE goal_event_state + SET high_watermark = high_watermark + 1, updated_at = ? WHERE goal_id = ? + RETURNING high_watermark AS sequence`).get(new Date().toISOString(), goalId) as { sequence: number } | undefined; + if (!row) throw new GoalSessionContractError( + 'Authoritative goal event allocator is missing', 'AUTHORITATIVE_DOMAIN_MISSING', + ); + return row.sequence; + } + + private readState(identity: GoalSessionIdentity): GoalSessionState | null { + const row = this.database.prepare('SELECT payload_json FROM goal_session_runtime_state WHERE scope = ?') + .get(sqliteGoalScope(identity)) as { payload_json: string } | undefined; + return row ? JSON.parse(row.payload_json) as GoalSessionState : null; + } + + private readEffect(identity: GoalProviderOperationFence, stage: GoalProviderEffectStage): EffectRow | undefined { + return this.database.prepare(`SELECT kind, status, claim_token, outcome_json FROM goal_session_runtime_provider_effects + WHERE scope = ? AND operation_id = ? AND stage = ?`) + .get(sqliteGoalScope(identity), identity.operationId, stage) as EffectRow | undefined; + } + + private readModelChange(identity: GoalSessionIdentity, operationId: string): GoalModelChangeHistoryRecord | undefined { + const row = this.database.prepare(`SELECT sequence, model, status, acknowledgement_json + FROM goal_session_runtime_model_changes WHERE scope = ? AND operation_id = ?`) + .get(sqliteGoalScope(identity), operationId) as { + sequence: number; model: string; status: GoalModelChangeHistoryRecord['status']; acknowledgement_json: string | null; + } | undefined; + return row ? { + operationId, sequence: row.sequence, model: row.model, status: row.status, + acknowledgement: row.acknowledgement_json + ? JSON.parse(row.acknowledgement_json) as GoalModelChangeAcknowledgement : undefined, + } : undefined; + } + + private assertOwner(identity: GoalSessionIdentity): void { + const row = this.database.prepare('SELECT goal_id FROM goal_provider_sessions WHERE session_id = ?') + .get(identity.sessionId) as { goal_id: string } | undefined; + if (!row || row.goal_id !== identity.goalId) throw new GoalSessionScopeError(); + } + + private bindOwnerForCreate(state: Omit): void { + if (!isSafeIdentifier(state.goalId) || !isSafeIdentifier(state.sessionId) + || !isSafeIdentifier(state.provider)) throw new GoalSessionScopeError(); + try { + this.database.prepare(`INSERT OR IGNORE INTO goal_provider_sessions + (session_id, goal_id, agent, effective_model, lease_generation, created_at, updated_at) + SELECT ?, goal_id, agent, effective_model, lease_epoch, ?, ? FROM goals + WHERE goal_id = ? AND agent = ?`) + .run(state.sessionId, new Date().toISOString(), new Date().toISOString(), state.goalId, state.provider); + } catch { throw new GoalSessionScopeError(); } + this.assertOwner(state); + } + + private hasCommit(kind: string, identity: string): boolean { + return Boolean(this.database.prepare('SELECT 1 FROM goal_session_runtime_commits WHERE kind = ? AND identity = ?').get(kind, identity)); + } + + private addCommit(owner: GoalSessionIdentity, kind: string, identity: string): void { + this.database.prepare(`INSERT INTO goal_session_runtime_commits + (session_id, goal_id, kind, identity) VALUES (?, ?, ?, ?)`) + .run(owner.sessionId, owner.goalId, kind, identity); + } + + private immediate(operation: () => T): T { + return this.database.transaction(operation).immediate(); + } +} + +/** Mandatory production composition; no ephemeral fallback is accepted. */ +export function createSqliteGoalSessionRuntimePorts( + database: Database.Database, + recovery: GoalSessionRecoveryPort, +): GoalSessionRuntimePorts { + const domain = new SqliteGoalSessionControlDomain(database); + return new AuthoritativeGoalSessionRuntimePorts(domain, recovery).asRuntimePorts(); +} + +function matchesControl(state: GoalSessionState | null, fence: GoalSessionControlFence): state is GoalSessionState { + return Boolean(state && state.controllerEpoch === fence.controllerEpoch + && state.providerBarrierIntent?.phase !== 'pending' + && !['cancelling', 'terminated', 'failed'].includes(state.status)); +} + +function matchesTurn( + state: GoalSessionState | null, + fence: GoalSessionFence, + execution: GoalExecutionIdentity, +): state is GoalSessionState { + return Boolean(matchesControl(state, fence) + && state.activeTurn?.turnId === fence.turnId + && state.activeTurn.executionId === execution.executionId + && state.activeTurn.attemptId === execution.attemptId + && !['completed', 'cancelled', 'failed'].includes(state.activeTurn.status)); +} + +function matchesTransition( + state: GoalSessionState | null, + transition: GoalSessionControlTransition, +): state is GoalSessionState { + if (!matchesControl(state, transition.fence)) return false; + return transition.turnScoped !== true + || 'turnId' in transition.fence && matchesTurn(state, transition.fence, transition.execution); +} + +function boundedEventKey(parts: readonly string[]): string { + return `goal-session:${createHash('sha256').update(parts.join('\0')).digest('hex')}`; +} + +function isPersistedRuntimeEvent( + value: unknown, + identity: GoalSessionIdentity, +): value is PersistedGoalSessionEvent { + if (!value || typeof value !== 'object' || Array.isArray(value) + || ![Object.prototype, null].includes(Object.getPrototypeOf(value))) return false; + const event = value as Partial; + if (event.goalId !== identity.goalId || event.sessionId !== identity.sessionId + || !isSafeIdentifier(event.turnId) || !isSafeIdentifier(event.executionId) + || !isSafeIdentifier(event.attemptId) || !Number.isSafeInteger(event.controllerEpoch) + || !Number.isSafeInteger(event.sequence) || (event.sequence ?? 0) < 1 + || typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt)) + || !event.event || typeof event.event !== 'object' || Array.isArray(event.event)) return false; + try { sanitizeGoalSessionEvent(event.event as GoalSessionEvent); } + catch { return false; } + return true; +} + +function effectCasFailure(): GoalSessionContractError { + return new GoalSessionContractError( + 'Provider effect token no longer owns the exact durable state', 'PROVIDER_EFFECT_IN_DOUBT', + ); +} diff --git a/packages/core/src/agents/goalSession/codexAppServer0146Bindings.generated.ts b/packages/core/src/agents/goalSession/codexAppServer0146Bindings.generated.ts new file mode 100644 index 000000000..165cdcaaf --- /dev/null +++ b/packages/core/src/agents/goalSession/codexAppServer0146Bindings.generated.ts @@ -0,0 +1,137 @@ +// GENERATED FROM `codex-cli 0.146.0 app-server generate-ts --experimental`. +// Keep this consumed protocol surface synchronized with the live-binary schema attestation test. + +export const CODEX_CLI_VERSION_0146 = '0.146.0'; +export const CODEX_APP_SERVER_PROTOCOL_0146 = 'app-server-0.146.0'; + +export const CODEX_APP_SERVER_METHODS_0146 = Object.freeze({ + initialize: 'initialize', + initialized: 'initialized', + modelList: 'model/list', + threadStart: 'thread/start', + threadResume: 'thread/resume', +}); + +export type CodexSandboxMode0146 = 'read-only' | 'workspace-write' | 'danger-full-access'; +export type CodexSubAgentSource0146 = + | 'review' + | 'compact' + | 'memory_consolidation' + | { other: string } + | { thread_spawn: { + parent_thread_id: string; depth: number; agent_path: string | null; + agent_nickname: string | null; agent_role: string | null; + } }; +export type CodexSessionSource0146 = + | 'cli' + | 'vscode' + | 'exec' + | 'appServer' + | { custom: string } + | { subAgent: CodexSubAgentSource0146 } + | 'unknown'; + +export interface CodexInitializeResponse0146 { + userAgent: string; + codexHome: string; + platformFamily: string; + platformOs: string; +} + +export interface CodexModelListResponse0146 { + data: unknown[]; + nextCursor: string | null; +} + +export interface CodexThread0146 { + id: string; + extra: Record | null; + sessionId: string; + forkedFromId: string | null; + parentThreadId: string | null; + preview: string; + ephemeral: boolean; + isPinned: boolean; + historyMode: 'legacy' | 'paginated'; + modelProvider: string; + createdAt: number; + updatedAt: number; + recencyAt: number | null; + status: unknown; + path: string | null; + cwd: string; + cliVersion: string; + source: CodexSessionSource0146; + canAcceptDirectInput: boolean | null; + threadSource: string | null; + agentNickname: string | null; + agentRole: string | null; + gitInfo: unknown | null; + name: string | null; + turns: unknown[]; +} + +export interface CodexThreadStartParams0146 { + model?: string | null; + modelProvider?: string | null; + allowProviderModelFallback?: boolean; + serviceTier?: string | null; + cwd?: string | null; + runtimeWorkspaceRoots?: string[] | null; + approvalPolicy?: 'untrusted' | 'on-request' | 'never' | Record | null; + approvalsReviewer?: 'user' | 'auto_review' | 'guardian_subagent' | null; + sandbox?: CodexSandboxMode0146 | null; + permissions?: string | null; + config?: Record | null; + serviceName?: string | null; + baseInstructions?: string | null; + developerInstructions?: string | null; + personality?: unknown | null; + multiAgentMode?: unknown | null; + ephemeral?: boolean | null; + historyMode?: 'legacy' | 'paginated' | null; + sessionStartSource?: unknown | null; + threadSource?: string | null; + environments?: unknown[] | null; + dynamicTools?: unknown[] | null; + selectedCapabilityRoots?: unknown[] | null; + mockExperimentalField?: string | null; + experimentalRawEvents?: boolean; +} + +export interface CodexThreadResumeParams0146 { + threadId: string; + history?: unknown[] | null; + path?: string | null; + model?: string | null; + modelProvider?: string | null; + serviceTier?: string | null; + cwd?: string | null; + runtimeWorkspaceRoots?: string[] | null; + approvalPolicy?: 'untrusted' | 'on-request' | 'never' | Record | null; + approvalsReviewer?: 'user' | 'auto_review' | 'guardian_subagent' | null; + sandbox?: CodexSandboxMode0146 | null; + permissions?: string | null; + config?: Record | null; + baseInstructions?: string | null; + developerInstructions?: string | null; + personality?: unknown | null; + excludeTurns?: boolean; + initialTurnsPage?: unknown | null; +} + +export interface CodexThreadResponse0146 { + thread: CodexThread0146; + model: string; + modelProvider: string; + serviceTier: string | null; + cwd: string; + runtimeWorkspaceRoots: string[]; + instructionSources: string[]; + approvalPolicy: unknown; + approvalsReviewer: unknown; + sandbox: unknown; + activePermissionProfile: unknown | null; + reasoningEffort: unknown | null; + multiAgentMode: unknown; +} diff --git a/packages/core/src/agents/goalSession/codexAppServer0146Validation.ts b/packages/core/src/agents/goalSession/codexAppServer0146Validation.ts new file mode 100644 index 000000000..62e64a970 --- /dev/null +++ b/packages/core/src/agents/goalSession/codexAppServer0146Validation.ts @@ -0,0 +1,101 @@ +import type { + CodexSessionSource0146, CodexThread0146, CodexThreadResponse0146, +} from './codexAppServer0146Bindings.generated.js'; + +export function assertExactThreadResponseFields(response: CodexThreadResponse0146): void { + if (typeof response.model !== 'string' || typeof response.modelProvider !== 'string' + || typeof response.cwd !== 'string' || !strings(response.runtimeWorkspaceRoots) + || !strings(response.instructionSources) + || !nullableString(response.serviceTier) + || !['user', 'auto_review', 'guardian_subagent'].includes(String(response.approvalsReviewer)) + || !validApprovalPolicy(response.approvalPolicy) || !plainObject(response.sandbox) + || !(response.activePermissionProfile === null || plainObject(response.activePermissionProfile)) + || !nullableString(response.reasoningEffort) + || !validMultiAgentMode(response.multiAgentMode)) malformed(); +} + +export function assertExactThreadFields(thread: CodexThread0146): void { + if (!validThreadStrings(thread) || !validThreadScalars(thread) || !validThreadNullableFields(thread) + || !(thread.extra === null || plainObject(thread.extra) && Object.keys(thread.extra).length === 0) + || !(thread.gitInfo === null || plainObject(thread.gitInfo)) + || !Array.isArray(thread.turns) || !validSessionSource(thread.source)) malformed(); +} + +function validThreadStrings(thread: CodexThread0146): boolean { + return string(thread.id) && string(thread.sessionId) && string(thread.preview) + && string(thread.modelProvider) && string(thread.cwd) && string(thread.cliVersion); +} + +function validThreadScalars(thread: CodexThread0146): boolean { + return typeof thread.ephemeral === 'boolean' && typeof thread.isPinned === 'boolean' + && ['legacy', 'paginated'].includes(thread.historyMode) + && finite(thread.createdAt) && finite(thread.updatedAt) + && (thread.recencyAt === null || finite(thread.recencyAt)) && validThreadStatus(thread.status) + && (thread.canAcceptDirectInput === null || typeof thread.canAcceptDirectInput === 'boolean'); +} + +function validThreadNullableFields(thread: CodexThread0146): boolean { + return nullableString(thread.path) && nullableString(thread.forkedFromId) + && nullableString(thread.parentThreadId) && nullableString(thread.threadSource) + && nullableString(thread.agentNickname) && nullableString(thread.agentRole) && nullableString(thread.name); +} + +function validSessionSource(value: unknown): value is CodexSessionSource0146 { + if (typeof value === 'string') return ['cli', 'vscode', 'exec', 'appServer', 'unknown'].includes(value); + if (!plainObject(value)) return false; + const keys = Object.keys(value); + if (keys.length !== 1) return false; + if (keys[0] === 'custom') return string(value.custom); + return keys[0] === 'subAgent' && validSubAgentSource(value.subAgent); +} + +function validSubAgentSource(value: unknown): boolean { + if (typeof value === 'string') return ['review', 'compact', 'memory_consolidation'].includes(value); + if (!plainObject(value) || Object.keys(value).length !== 1) return false; + if ('other' in value) return string(value.other); + if (!('thread_spawn' in value) || !plainObject(value.thread_spawn)) return false; + const spawn = value.thread_spawn; + return exactKeys(spawn, ['parent_thread_id', 'depth', 'agent_path', 'agent_nickname', 'agent_role']) + && string(spawn.parent_thread_id) && Number.isSafeInteger(spawn.depth) && Number(spawn.depth) >= 0 + && nullableString(spawn.agent_path) && nullableString(spawn.agent_nickname) + && nullableString(spawn.agent_role); +} + +function validThreadStatus(value: unknown): boolean { + if (!plainObject(value) || typeof value.type !== 'string') return false; + if (value.type === 'active') return exactKeys(value, ['type', 'activeFlags']) && Array.isArray(value.activeFlags); + return ['notLoaded', 'idle', 'systemError'].includes(value.type) && exactKeys(value, ['type']); +} + +function validApprovalPolicy(value: unknown): boolean { + if (typeof value === 'string') return ['untrusted', 'on-request', 'never'].includes(value); + if (!plainObject(value) || !plainObject(value.granular) || !exactKeys(value, ['granular'])) return false; + const granular = value.granular; + const fields = ['sandbox_approval', 'rules', 'skill_approval', 'request_permissions', 'mcp_elicitations']; + return exactKeys(granular, fields) && fields.every(field => typeof granular[field] === 'boolean'); +} + +function validMultiAgentMode(value: unknown): boolean { + if (value === 'explicitRequestOnly' || value === 'proactive') return true; + return plainObject(value) && exactKeys(value, ['custom']) && string(value.custom); +} + +function plainObject(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function exactKeys(value: Record, keys: string[]): boolean { + const actual = Object.keys(value); + return actual.length === keys.length && keys.every(key => key in value); +} + +function strings(value: unknown): value is string[] { + return Array.isArray(value) && value.every(string); +} + +function string(value: unknown): value is string { return typeof value === 'string'; } +function nullableString(value: unknown): boolean { return value === null || string(value); } +function finite(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value); } +function malformed(): never { throw new Error('App Server response violates generated Codex 0.146 bindings'); } diff --git a/packages/core/src/agents/goalSession/contract.ts b/packages/core/src/agents/goalSession/contract.ts new file mode 100644 index 000000000..baf762af2 --- /dev/null +++ b/packages/core/src/agents/goalSession/contract.ts @@ -0,0 +1,639 @@ +import type { + GoalModelInvocationEvidence, GoalProviderBarrierIntent, GoalProviderOpenContext, + GoalProviderOperationFence, GoalUsageAccounting, +} from './providerOperationBoundary.js'; +import type { GoalProviderCapabilities } from './providerCapabilities.js'; +export type { + GoalModelChangeHistoryPort, GoalModelChangeHistoryRecord, GoalModelInvocationEvidence, + GoalProviderBarrierIntent, GoalProviderBarrierPublication, GoalProviderDuplexTransport, + GoalProviderEffectStage, GoalProviderFirstEffectPort, GoalProviderOpenContext, GoalProviderOperationFence, GoalStartedProviderEffect, GoalStartedProviderEffectCleanup, GoalUsageAccounting, +} from './providerOperationBoundary.js'; +export type { + GoalModelChangeBoundary, GoalNativeSessionIdTiming, GoalPauseBoundary, + GoalProviderCapabilities, GoalSteeringBoundary, +} from './providerCapabilities.js'; +export type { GoalSessionRecoveryPort, GoalSessionRuntimePorts } from './runtimePorts.js'; + +/** JSON values are used for recovery data so it can be persisted without provider objects. */ +export type GoalSessionJsonValue = + | string + | number + | boolean + | null + | GoalSessionJsonValue[] + | { [key: string]: GoalSessionJsonValue }; + +export interface GoalSessionIdentity { + goalId: string; + /** ProPR's stable session identity. This is distinct from a provider session ID. */ + sessionId: string; +} + +/** + * Session-scoped ownership fence. Control operations (pause, resume, model + * change, cancel, reconcile) are authorized by goal/session/epoch alone and do + * not require an active turn. This is deliberately separate from the turn fence + * so control/audit events can be appended even when no turn is running. + */ +export interface GoalSessionControlFence extends GoalSessionIdentity { + /** Monotonically increasing ownership generation. */ + controllerEpoch: number; +} + +/** Turn-scoped fence. Adds the specific logical turn a caller claims to own. */ +export interface GoalSessionFence extends GoalSessionControlFence { + turnId: string; +} + +export interface GoalRepositoryIdentity { + repository: string; + worktreePath: string; + branch: string; + /** Mutable checkout checkpoint for diagnostics/resume, never repository identity. */ + headSha?: string; +} + +export interface GoalExecutionIdentity { + /** Stable across queue redelivery of this turn. */ + executionId: string; + /** Unique for an actual provider invocation, including a recovered retry. */ + attemptId: string; +} + +export type GoalSessionStatus = + | 'initializing' + | 'idle' + | 'running' + | 'pause_requested' + | 'paused' + | 'cancelling' + | 'terminated' + | 'failed'; + +export interface GoalTurnState extends GoalExecutionIdentity { + turnId: string; + /** Controller epoch that started this concrete provider invocation. */ + executionEpoch: number; + objective: string; + requestedModel: string; + repository: GoalRepositoryIdentity; + /** Cancellation/replacement barrier captured for this provider invocation. */ + providerOperationGeneration?: number; + /** Exact deferred model generation this invocation is entitled to apply. */ + modelChange?: { modelChangeId: string; generation: number; previousModel?: string }; + status: 'running' | 'pause_requested' | 'paused' | 'completed' | 'cancelled' | 'failed'; +} + +/** + * Durable record of a finished logical turn's real execution identity. It lets a + * late queue redelivery of an older turn recover the exact execution/attempt it + * ran under, instead of fabricating a fresh attempt id, even after a subsequent + * turn has replaced {@link GoalSessionState.activeTurn}. + */ +export interface GoalCompletedTurn extends GoalExecutionIdentity { + turnId: string; +} + +/** + * Durable marker recorded before the first provider initialization/open call. It lets a + * later controller distinguish an ordinary crash window (recoverable when the + * provider can deterministically/idempotently re-open) from a session that was + * never intended to be initialized. + */ +export interface GoalSessionInitializationIntent { + attemptId: string; + /** Stable key a deterministic provider uses to re-open or retry the same initialization. */ + deterministicOpenKey: string; + recordedAt: string; +} + +/** + * Durable claim for one adapter reconciliation call. It is intentionally + * separate from activeTurn: until the adapter reports that it enacted a + * replacement, the pre-crash attempt remains the authoritative live identity. + */ +export interface GoalRecoveryAttempt { + /** Stable idempotency/fencing identity for this recovery provider operation. */ + operationToken: string; + /** Monotonic durable provider fence. A provider must reject a lower generation. */ + operationGeneration: number; + executionId: string; + attemptId: string; + controllerEpoch: number; + authoritativeAttemptId?: string; + authoritativeExecutionId?: string; + /** Exact live status captured by the durable claim. */ + sessionStatus?: GoalSessionStatus; + authoritativeTurnStatus?: GoalTurnState['status']; + claimedAt: string; + /** A replacement may reclaim an abandoned operation only after this durable lease expires. */ + leaseExpiresAt: string; + /** Claimed work is cancellation-preemptible until the provider call is durably marked in doubt. */ + phase?: 'claimed' | 'provider_in_doubt'; +} + +export type GoalResumeKind = 'active_turn' | 'after_turn' | 'recovered_after_turn'; + +/** Durable exclusive claim around one logical resume provider operation. */ +export interface GoalResumeIntent extends GoalExecutionIdentity { + operationId: string; + operationGeneration: number; + kind: GoalResumeKind; + controllerEpoch: number; + turnId?: string; + claimedAt: string; + leaseExpiresAt: string; + phase: 'claimed' | 'provider_in_doubt' | 'settled'; +} + +export interface GoalCompletedResume { + operationId: string; + operationGeneration: number; + kind: GoalResumeKind; + controllerEpoch: number; +} + +/** Atomic recovery-result receipt used to replay an ambiguous committed transaction. */ +export interface GoalCompletedRecovery { + operationToken: string; + controllerEpoch: number; + outcome: 'alive' | 'resumed' | 'failed'; + reason: string; +} + +/** Durable cancellation claim recorded before the provider cancellation side effect. */ +export interface GoalCancellationIntent { + /** Stable provider idempotency identity, retained through terminal recovery. */ + cancellationId: string; + reason: string; + claimedAt: string; + /** Captured before activeTurn is cleared so lazy-ID cancellation can target the old invocation. */ + pendingContext?: GoalPendingCancellationContext; +} + +/** Durable model side-effect intent recorded before calling the provider. */ +export interface GoalModelChangeIntent { + /** Stable provider idempotency identity used for every recovery retry. */ + modelChangeId: string; + model: string; + requestedAt: string; + /** Monotonic session-local ordering for external provider application. */ + generation?: number; + /** Stable audit predecessor captured when this generation is accepted. */ + previousModel?: string; + /** Durable provider-call phase; missing is treated as pending for older records. */ + phase?: 'pending' | 'provider_in_doubt' | 'committed' | 'superseded_in_doubt' | 'superseded'; + /** Unique lease owner for one generation-scoped provider application attempt. */ + applicationToken?: string; + /** Controller generation that owns applicationToken. */ + applicationControllerEpoch?: number; + /** Durable recovery deadline for a process that disappears during provider application. */ + leaseExpiresAt?: string; + /** Retained after commit so an ambiguous retry can return the original acknowledgement. */ + acknowledgement?: GoalModelChangeAcknowledgement; + /** Exact first-invocation evidence for a next-turn model application. */ + invocationEvidence?: GoalModelInvocationEvidence; + /** Prior-attempt occurrence retained only to dedupe exact recovery replay. */ + previousInvocationEvidence?: GoalModelInvocationEvidence; +} + +export interface GoalProviderSessionSnapshot { + /** Stable, provider-issued identity. It must never be replaced during resume. */ + providerSessionId: string; + /** Serializable, credential-free provider checkpoint/recovery metadata. */ + recoveryMetadata: GoalSessionJsonValue; + model?: string; +} + +/** Provider context for a turn; pending identity never contains a fake native ID. */ +export type GoalProviderTurnContext = + | { binding: 'bound'; snapshot: GoalProviderSessionSnapshot } + | { binding: 'pending'; initializationIntent: GoalSessionInitializationIntent }; + +export interface GoalSessionState extends GoalSessionIdentity { + provider: string; + providerSessionId?: string; + recoveryMetadata?: GoalSessionJsonValue; + controllerEpoch: number; + status: GoalSessionStatus; + currentModel?: string; + requestedModel?: string; + /** Deferred model request awaiting the provider's declared next-turn boundary. */ + pendingModelChange?: string; + /** Durable obligation to record an after-turn pause boundary with completion. */ + pendingAfterTurnPause?: boolean; + activeTurn?: GoalTurnState; + completedTurnIds: string[]; + /** Execution identity of each completed turn, keyed by turnId order of completion. */ + completedTurns?: GoalCompletedTurn[]; + /** Present while a first provider open is in-flight; cleared once persisted. */ + initializationIntent?: GoalSessionInitializationIntent; + /** Last durably claimed provider open/resume invocation attempt. */ + providerOpenAttemptId?: string; providerOpenOperationGeneration?: number; + /** A crashed first-turn invocation that may be retried with a fresh attempt. */ + retryTurn?: { turnId: string; executionId: string; crashedAttemptId: string }; + /** Last durably claimed reconciliation invocation attempt. */ + recoveryAttemptId?: string; + /** In-flight reconciliation claim, retained across a thrown call or crash. */ + recoveryAttempt?: GoalRecoveryAttempt; + /** Last atomically committed reconciliation receipt for same-epoch replay. */ + completedRecovery?: GoalCompletedRecovery; + /** Last allocated generation across recovery/resume provider operations. */ + providerOperationGeneration?: number; + /** Blocks new primitives while an invalidating publication is pending. */ + providerBarrierIntent?: GoalProviderBarrierIntent; + resumeIntent?: GoalResumeIntent; + completedResume?: GoalCompletedResume; + /** In-flight or completed cancellation identity. Active turn ownership is cleared when this is claimed. */ + cancellationIntent?: GoalCancellationIntent; + /** Provider model application/reconciliation identity retained across crashes. */ + modelChangeIntent?: GoalModelChangeIntent; + /** + * Ordered immediate-model generations. Unresolved work and a bounded settled + * retry window are retained; canonical audit history lives in the event log. + */ + modelChangeIntents?: GoalModelChangeIntent[]; + /** Last allocated immediate-model generation. */ + modelChangeGeneration?: number; + /** Bounded retry-safe provider usage cursor. */ + usageAccounting?: GoalUsageAccounting; + failureReason?: string; + /** Optimistic concurrency token owned by the state port. */ + version: number; + createdAt: string; + updatedAt: string; +} + +export type GoalToolPhase = 'started' | 'progress' | 'completed' | 'failed'; + +export type GoalSessionEvent = + | { type: 'output'; channel: 'stdout' | 'stderr'; data: string } + | { type: 'assistant'; messageId?: string; content: string; data?: GoalSessionJsonValue } + | { type: 'tool'; toolCallId: string; name: string; phase: GoalToolPhase; data?: GoalSessionJsonValue } + | { type: 'todo'; todoId: string; title: string; status: 'pending' | 'in_progress' | 'completed' | 'cancelled'; data?: GoalSessionJsonValue } + | { + type: 'usage'; occurrenceId: string; semantics: 'delta' | 'cumulative'; watermark: number; + model?: string; inputTokens?: number; outputTokens?: number; + cachedInputTokens?: number; costUsd?: number; data?: GoalSessionJsonValue; + } + | { type: 'checkpoint'; checkpointId: string; recoveryMetadata: GoalSessionJsonValue; providerSessionId?: string } + | { type: 'message_acknowledged'; messageId: string } + | { type: 'pause_requested'; appliesAt: 'immediate' | 'next_safe_boundary' | 'after_turn' } + | { type: 'pause_boundary'; boundary: string; checkpointId?: string; providerEventId?: string; providerEventOrdinal?: number } + | { type: 'session_resumed' } + | { type: 'model_change_acknowledged'; requestedModel: string; appliesAt: 'immediate' | 'next_safe_boundary' | 'next_turn' } + | { type: 'model_changed'; previousModel?: string; model: string; providerEventId?: string; providerEventOrdinal?: number } + | { type: 'turn_resumed'; turnId: string } + | { type: 'reconciliation'; outcome: 'alive' | 'resumed' | 'failed' | 'blocked'; reason: string } + | { type: 'completion'; outcome: 'succeeded' | 'failed' | 'cancelled'; summary?: string; error?: string }; + +/** Required occurrence identity contract for provider-streamed atomic transitions. */ +export type GoalProviderStreamTransitionEvent = Extract< + GoalSessionEvent, + { type: 'model_changed' | 'pause_boundary' } +> & ( + | { providerEventId: string; providerEventOrdinal?: number } + | { providerEventId?: undefined; providerEventOrdinal: number } +); + +export interface PersistedGoalSessionEvent extends GoalSessionFence, GoalExecutionIdentity { + sequence: number; + recordedAt: string; + event: GoalSessionEvent; +} + +export type GoalEventAppendResult = + | { accepted: true; persisted: PersistedGoalSessionEvent } + | { accepted: false; reason: 'stale_fence' | 'wrong_goal' | 'turn_not_active' }; + +/** + * State changes are compare-and-swap operations. Implementations must scope a + * sessionId to its original goalId and reject cross-goal reads or writes. + */ +export interface GoalSessionStatePort { + load(identity: GoalSessionIdentity): Promise; + create(state: Omit): Promise; + compareAndSet(expected: GoalSessionState, next: Omit): Promise; +} + +export interface GoalSessionControlTransition { + /** Stable idempotency identity for ambiguous post-commit recovery. */ + transitionId: string; + /** A turn fence makes the transaction authoritative only for the exact live invocation. */ + fence: GoalSessionControlFence | GoalSessionFence; + /** Explicit because a control request may carry an ignored excess turnId property at runtime. */ + turnScoped?: true; + execution: GoalExecutionIdentity; + auditEvents: ReadonlyArray>; +} + +/** Atomically commits nonterminal state and its canonical ordered audit events. */ +export interface GoalSessionTransitionPort { + commit( + expected: GoalSessionState, + next: Omit, + transition: GoalSessionControlTransition, + ): Promise; +} + +export type GoalTerminalCommit = + | { + scope: 'turn'; + fence: GoalSessionFence; + execution: GoalExecutionIdentity; + /** Ordered audit events committed immediately before terminal completion. */ + auditEvents: ReadonlyArray>; + event: Extract; + } + | { + scope: 'control'; + fence: GoalSessionControlFence; + execution: GoalExecutionIdentity; + /** Ordered audit events committed immediately before terminal completion. */ + auditEvents: ReadonlyArray>; + event: Extract; + }; + +/** + * Commits terminal state and its ordered audit/completion events in one durable transaction. + * Implementations must be idempotent by scope/fence/execution, so an ambiguous + * post-commit transport failure can be retried without a duplicate event. + */ +export interface GoalSessionTerminalPort { + commit( + expected: GoalSessionState, + next: Omit, + completion: GoalTerminalCommit, + ): Promise; +} + +/** + * An append is authoritative only when goal/session/epoch/turn still match. + * The fence check and append must be one atomic durable operation. Appending a + * delta (rather than replacing a snapshot) preserves replay across restarts. + */ +export interface GoalSessionEventSink { + /** + * Turn-scoped append. Authoritative only when goal/session/epoch match AND + * the fence owns the currently active turn. Rejects output attributed to a + * turn that is not active (including terminal turns). + */ + append(fence: GoalSessionFence, execution: GoalExecutionIdentity, event: GoalSessionEvent): Promise; + /** + * Session-scoped control/audit append. Authoritative when goal/session/epoch + * match; it does not require an active turn, so model-change, cancel, resume + * and reconciliation remain auditable in idle state. It must never be + * attributed to an unrelated completed turn. + */ + appendControl(fence: GoalSessionControlFence, execution: GoalExecutionIdentity, event: GoalSessionEvent): Promise; + replay(identity: GoalSessionIdentity, afterSequence?: number): Promise; +} + +export interface DurableCorrectiveMessage extends GoalSessionIdentity { + messageId: string; + sequence: number; + body: string; + createdAt: string; + acknowledgedAt?: string; +} + +/** Message creation belongs to goal persistence/API code; the runtime only consumes and acknowledges it. */ +export interface GoalSessionMessagePort { + listPending(identity: GoalSessionIdentity): Promise; + /** Atomically consumes the message and appends its canonical acknowledgement event. */ + acknowledgeWithEvent( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + ): Promise<'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found'>; +} + +export interface GoalProviderOpenRequest extends GoalSessionIdentity { + provider: string; controllerEpoch: number; attemptId: string; + operationGeneration: number; operationFence: GoalProviderOperationFence; + persisted?: GoalProviderSessionSnapshot; + /** + * Stable key a deterministic provider uses to re-open the same underlying + * session after a crash that happened before the provider identity was + * persisted. Only meaningful when the adapter reports supportsDeterministicOpen. + */ + deterministicOpenKey?: string; + /** Required by eager process-like providers such as Codex App Server. */ + openContext?: GoalProviderOpenContext; +} + +export interface GoalBeginTurnRequest extends GoalSessionFence, GoalExecutionIdentity { + objective: string; + context?: GoalSessionJsonValue; + repository: GoalRepositoryIdentity; + requestedModel: string; + operationGeneration: number; + operationFence: GoalProviderOperationFence; + /** + * FIFO messages reserved for acceptance by a next-turn-only provider. The + * provider must acknowledge every supplied ID before reporting success. + */ + correctiveMessages?: GoalProviderCorrectiveMessage[]; + /** Present when this invocation settles a durable recovered-resume claim. */ + providerOperation?: Pick; + /** Deferred model intent applied by this invocation, never by a pre-turn side call. */ + modelChange?: { modelChangeId: string; generation: number }; +} + +export interface GoalProviderCorrectiveMessage { + messageId: string; sequence: number; body: string; +} + +/** Legacy-compatible supervisor command; the provider never receives this weaker shape. */ +export interface GoalSteeringCommand extends GoalSessionFence { + executionId?: string; + attemptId?: string; + messageId: string; + body: string; +} + +export interface GoalSteeringRequest extends GoalSessionFence, GoalExecutionIdentity { + messageId: string; body: string; + operationGeneration: number; operationFence: GoalProviderOperationFence; +} + +export interface GoalPauseRequest extends GoalSessionControlFence { + reason?: string; + operationGeneration?: number; + operationFence?: GoalProviderOperationFence; +} + +export interface GoalModelChangeRequest extends GoalSessionControlFence { + model: string; + /** Stable caller identity for direct retry within the supported durable horizon. */ + operationId?: string; +} + +/** Provider request; retries with the same modelChangeId must not repeat the external side effect. */ +export interface GoalProviderModelChangeRequest extends GoalModelChangeRequest { + modelChangeId: string; + /** + * Durable monotonic application order. Providers must fence older generations + * after observing a newer one, including delayed completion of an older call. + */ + applicationGeneration: number; + operationGeneration: number; + operationFence: GoalProviderOperationFence; +} + +export interface GoalCancelRequest extends GoalSessionControlFence { + reason: string; +} + +/** Provider request; retries with the same cancellationId must be idempotent. */ +export interface GoalProviderCancelRequest extends GoalCancelRequest { + cancellationId: string; + operationGeneration: number; + operationFence: GoalProviderOperationFence; +} +/** Identity available while a lazy-ID provider has not emitted its first checkpoint. */ +export interface GoalPendingCancellationContext { + initializationIntent: GoalSessionInitializationIntent; + activeTurn?: Pick; +} + +export interface GoalPauseAcknowledgement { + appliesAt: 'immediate' | 'next_safe_boundary' | 'after_turn'; + /** Present when the control call itself reached the boundary; otherwise the turn stream reports it later. */ + boundaryReached?: { boundary: string; checkpointId?: string }; +} + +export type GoalMessageDeliveryOutcome = + | { outcome: 'acknowledged'; messageId: string; acknowledgement: 'acknowledged' | 'already_acknowledged' } + | { outcome: 'unsupported_same_turn'; messageId: string; supportedBoundary: 'next_turn' }; + +export type GoalTurnResumeCapabilityOutcome = { + /** Operator pause cannot retain a resumable active invocation at this boundary. */ + disposition: 'unsupported_same_turn'; + supportedBoundary: 'after_turn'; +}; + +export interface GoalModelChangeAcknowledgement { + outcome?: 'acknowledged' | 'outside_retry_horizon'; requestedModel: string; + appliesAt: 'immediate' | 'next_safe_boundary' | 'next_turn'; effectiveModel?: string; +} + +export interface GoalProviderReconcileRequest extends GoalSessionIdentity, GoalExecutionIdentity { + controllerEpoch: number; + /** Durable recovery operation identity; retries/replacements must be fenced by the provider primitive. */ + operationToken: string; + operationGeneration: number; + operationPhase: 'provider_in_doubt'; + operationLeaseExpiresAt: string; + operationFence: GoalProviderOperationFence; + persisted: GoalProviderSessionSnapshot; + repository: GoalRepositoryInspection; + container: GoalContainerInspection; +} + +export interface GoalProviderResumeRequest extends GoalSessionControlFence { + operationId: string; + operationGeneration: number; + operationPhase: 'provider_in_doubt' | 'settled'; + operationLeaseExpiresAt: string; + kind: GoalResumeKind; + operationFence: GoalProviderOperationFence; +} + +export type GoalProviderReconcileResult = + | { outcome: 'alive'; snapshot?: GoalProviderSessionSnapshot; reason: string } + | { outcome: 'resumed'; snapshot: GoalProviderSessionSnapshot; reason: string } + | { outcome: 'failed'; reason: string }; + +/** + * Provider-specific CLI parsing and resume semantics live behind this adapter. + * Pause/model-change requests are immediate control calls, but their effects may + * be deferred as explicitly reported by the acknowledgement and later events. + * deliverMessage must be idempotent by messageId so crash retries do not steer twice. + * requestModelChange must be idempotent by modelChangeId and monotonically + * fenced by applicationGeneration so recovery after a provider-success/ + * persistence-crash window never applies one intent twice and a delayed older + * generation cannot overwrite a newer provider effect. + * cancel and cancelPending must likewise be idempotent by cancellationId because + * a crash can occur after signalling the provider but before the terminal + * transaction is observed by the caller. + * reconcile/resume primitives must durably reject an expired lease or an + * operationGeneration below the newest generation they have observed, before + * starting any provider side effect. operationId/token supplies idempotency; + * generation supplies replacement/cancellation ordering. + * Every non-open primitive reached through GoalProviderFirstEffectPort must be + * idempotent by its exact operation fence because an unresolved durable start + * is re-invoked after process recovery. A non-idempotent eager open is instead + * terminal/in-doubt unless its validated outcome was durably settled. + */ +export interface GoalSessionAdapter { + readonly provider: string; + readonly capabilities: GoalProviderCapabilities; + /** + * When true, openSession is idempotent for a given deterministicOpenKey: a + * repeated call re-opens the same provider session instead of creating a new + * one. This is what makes a crash before provider-identity persistence + * recoverable rather than permanently failed. + */ + readonly supportsDeterministicOpen?: boolean; + /** + * Publishes the session's monotonic high-water generation into the same + * durable provider/container/remote authority used by every primitive. + * The adapter must retain pendingCancellationId after caller timeout until + * that cancellation is known settled; advancing the work generation must + * never erase the only cancellation capable of stopping older work. + */ + publishOperationBarrier(publication: import('./providerOperationBoundary.js').GoalProviderBarrierPublication): Promise; + openSession(request: GoalProviderOpenRequest): Promise; + /** + * Every model_changed and pause_boundary occurrence must carry a non-empty + * providerEventId or a stable non-negative providerEventOrdinal. If both are + * present, providerEventId is the canonical occurrence identity. + */ + beginTurn(request: GoalBeginTurnRequest, context: GoalProviderTurnContext): AsyncIterable; + /** + * Continues the exact active turn identified by the fence after a pause or a + * container/supervisor restart. The provider resumes from the durable + * checkpoint and streams further ordered events through to a single + * completion; it must not start a new logical turn. + */ + resumeTurn?(request: GoalSessionFence & GoalExecutionIdentity & GoalProviderResumeRequest, snapshot: GoalProviderSessionSnapshot): AsyncIterable; + deliverMessage?(request: GoalSteeringRequest, snapshot: GoalProviderSessionSnapshot): Promise<{ messageId: string }>; + requestPause?(request: GoalPauseRequest, snapshot: GoalProviderSessionSnapshot): Promise; + resumeSession(request: GoalProviderResumeRequest, snapshot: GoalProviderSessionSnapshot): Promise; + requestModelChange(request: GoalProviderModelChangeRequest, snapshot: GoalProviderSessionSnapshot): Promise; + cancel(request: GoalProviderCancelRequest, snapshot: GoalProviderSessionSnapshot): Promise; + /** Cancels an invocation/container before a native provider session ID exists. */ + cancelPending?(request: GoalProviderCancelRequest, pending: GoalPendingCancellationContext): Promise; + reconcile(request: GoalProviderReconcileRequest): Promise; +} + +export type GoalContainerStatus = 'running' | 'exited' | 'missing' | 'daemon_unavailable'; + +export interface GoalContainerInspection { + status: GoalContainerStatus; + containerId?: string; + containerName?: string; + /** Authoritative labels read from the recovered container itself. */ + recoveryIdentity?: GoalRecoveryIdentity; + reason?: string; +} + +export interface GoalRecoveryIdentity extends GoalSessionIdentity { + executionEpoch: number; + turnId: string; + attemptId: string; + worktreeFingerprint: string; +} + +export interface GoalRepositoryInspection extends GoalRepositoryIdentity { + exists: boolean; + dirty?: boolean; + /** Repository URL/name read from Git rather than copied from the request. */ + observedRepository?: string; + observedHeadSha?: string; + observedBranch?: string; + observedWorktreeFingerprint?: string; + resolvedWorktreePath?: string; + reason?: string; +} diff --git a/packages/core/src/agents/goalSession/controlOperationIdentity.ts b/packages/core/src/agents/goalSession/controlOperationIdentity.ts new file mode 100644 index 000000000..37f1aabb5 --- /dev/null +++ b/packages/core/src/agents/goalSession/controlOperationIdentity.ts @@ -0,0 +1,26 @@ +import { createHash } from 'node:crypto'; +import type { GoalSessionState } from './contract.js'; +import { GoalSessionContractError } from './errors.js'; + +export function mintFreshAttemptId(previousAttemptId: string, mint: () => string): string { + for (let attempt = 0; attempt < 4; attempt += 1) { + const candidate = mint(); + if (candidate && candidate !== previousAttemptId) return candidate; + } + throw new GoalSessionContractError('Could not mint a fresh recovery attempt identity', 'RECOVERY_ATTEMPT_REUSED'); +} + +/** Stable, non-secret identity for a control operation claimed at one state version. */ +export function controlOperationId(kind: string, state: GoalSessionState): string { + const scope = createHash('sha256') + .update(`${state.goalId}\0${state.sessionId}`) + .digest('hex') + .slice(0, 24); + return `${kind}-${scope}-e${state.controllerEpoch}-v${state.version}`; +} + +/** Bounded stable identity for composites of independently bounded caller IDs. */ +export function compositeOperationId(kind: string, ...parts: readonly string[]): string { + const digest = createHash('sha256').update(parts.join('\0')).digest('hex'); + return `${kind}-${digest}`; +} diff --git a/packages/core/src/agents/goalSession/durableStateRelationships.ts b/packages/core/src/agents/goalSession/durableStateRelationships.ts new file mode 100644 index 000000000..5bf4e89c0 --- /dev/null +++ b/packages/core/src/agents/goalSession/durableStateRelationships.ts @@ -0,0 +1,350 @@ +import type { GoalModelChangeIntent, GoalSessionState } from './contract.js'; +import { GoalSessionContractError } from './errors.js'; + +/** Cross-field invariants applied only after every durable field is decoded. */ +export function validateStateRelationships(state: GoalSessionState): void { + validateStatusRelationships(state); + validateBarrierRelationships(state); + validateOperationGenerations(state); + validateStateCollections(state); + validateModelGenerations(state); +} + +function validateStatusRelationships(state: GoalSessionState): void { + if (hasInvalidStatusTurnRelationship(state)) invalid('status/activeTurn relationship'); + if (state.activeTurn && state.activeTurn.executionEpoch > state.controllerEpoch) invalid('activeTurn.executionEpoch'); + if (state.status === 'cancelling' && (!state.cancellationIntent || state.activeTurn !== undefined)) { + invalid('cancelling state'); + } + if (hasCancellationIntentInLiveStatus(state)) invalid('cancellationIntent status'); + if (hasAfterTurnPauseInInvalidStatus(state)) invalid('pendingAfterTurnPause'); + if (state.retryTurn && (state.activeTurn || state.status !== 'idle')) invalid('retryTurn'); +} + +function hasInvalidStatusTurnRelationship(state: GoalSessionState): boolean { + const turn = state.activeTurn; + const live = turn !== undefined && !['completed', 'cancelled', 'failed'].includes(turn.status); + switch (state.status) { + case 'running': return turn?.status !== 'running'; + case 'pause_requested': return turn?.status !== 'pause_requested'; + case 'paused': return turn !== undefined && turn.status !== 'paused'; + case 'idle': return live; + case 'initializing': return turn !== undefined; + case 'cancelling': + case 'terminated': + case 'failed': return live; + } +} + +function hasCancellationIntentInLiveStatus(state: GoalSessionState): boolean { + if (!state.cancellationIntent) return false; + return state.status !== 'cancelling' && state.status !== 'terminated' && state.status !== 'failed'; +} + +function hasAfterTurnPauseInInvalidStatus(state: GoalSessionState): boolean { + if (!state.pendingAfterTurnPause) return false; + return state.status !== 'running' && state.status !== 'pause_requested' && state.status !== 'paused'; +} + +function validateBarrierRelationships(state: GoalSessionState): void { + validateBarrierGeneration(state); + if (state.cancellationIntent?.pendingContext && state.providerSessionId) invalid('cancellationIntent.pendingContext'); + if (state.status === 'cancelling' && !state.cancellationIntent) invalid('cancellationIntent'); + validateCancellingBarrier(state); + validateCancellationBarrierIdentity(state); + validateBarrierStatus(state); + validatePendingCancellationKind(state); + validateTerminalCancellationIdentity(state); + validateLeaseExpiryIdentity(state); + if (state.initializationIntent && state.providerSessionId) invalid('initializationIntent'); +} + +function validateLeaseExpiryIdentity(state: GoalSessionState): void { + const barrier = state.providerBarrierIntent; + if (barrier?.kind !== 'lease_expiry' || barrier.phase !== 'pending') return; + const matchesResume = state.resumeIntent?.phase === 'provider_in_doubt' + && barrier.generation === state.resumeIntent.operationGeneration + 1 + && barrier.operationId === `${state.resumeIntent.operationId}:lease-expiry`; + const matchesRecovery = state.recoveryAttempt?.phase === 'provider_in_doubt' + && barrier.generation === state.recoveryAttempt.operationGeneration + 1 + && barrier.operationId === `${state.recoveryAttempt.operationToken}:lease-expiry`; + if (!matchesResume && !matchesRecovery) invalid('orphan lease_expiry barrier'); +} + +function validateBarrierGeneration(state: GoalSessionState): void { + const barrier = state.providerBarrierIntent; + if (!barrier) return; + const generation = state.providerOperationGeneration; + if (generation === undefined || barrier.generation > generation) invalid('providerBarrierIntent.generation'); + if (barrier.phase === 'pending' && barrier.generation !== generation) invalid('providerBarrierIntent.generation'); +} + +function validateCancellingBarrier(state: GoalSessionState): void { + if (state.status !== 'cancelling') return; + const barrier = state.providerBarrierIntent; + if (barrier?.kind !== 'cancellation' || barrier.generation !== state.providerOperationGeneration) { + invalid('cancelling barrier'); + } +} + +function validateCancellationBarrierIdentity(state: GoalSessionState): void { + const barrier = state.providerBarrierIntent; + if (barrier?.kind !== 'cancellation') return; + if (!state.cancellationIntent || barrier.pendingCancellationId !== state.cancellationIntent.cancellationId) { + invalid('providerBarrierIntent.pendingCancellationId'); + } +} + +function validateBarrierStatus(state: GoalSessionState): void { + const kind = state.providerBarrierIntent?.kind; + if (kind === 'cancellation' && state.status !== 'cancelling') invalid('cancellation barrier status'); + if (kind === 'terminal' && state.status !== 'terminated' && state.status !== 'failed') { + invalid('terminal barrier status'); + } +} + +function validatePendingCancellationKind(state: GoalSessionState): void { + const barrier = state.providerBarrierIntent; + if (barrier?.pendingCancellationId === undefined) return; + if (barrier.kind !== 'cancellation' && barrier.kind !== 'terminal') { + invalid('providerBarrierIntent.pendingCancellationId'); + } +} + +function validateTerminalCancellationIdentity(state: GoalSessionState): void { + const barrier = state.providerBarrierIntent; + if (barrier?.kind !== 'terminal') return; + if (!state.cancellationIntent || barrier.pendingCancellationId !== state.cancellationIntent.cancellationId) { + invalid('terminal pendingCancellationId'); + } +} + +function validateOperationGenerations(state: GoalSessionState): void { + validateRecoveryGeneration(state); + validateResumeGeneration(state); + if (state.resumeIntent && state.recoveryAttempt) invalid('resume/recovery overlap'); + if (state.recoveryAttempt && state.completedRecovery) invalid('recovery/completedRecovery overlap'); + validateRecoveryIdentity(state); + validateCompletedResume(state); + validateResumeTurn(state); + validateProviderOpenGeneration(state); + validateActiveTurnGeneration(state); +} + +function validateRecoveryGeneration(state: GoalSessionState): void { + const recovery = state.recoveryAttempt; + if (!recovery) return; + if (recovery.controllerEpoch > state.controllerEpoch + || recovery.operationGeneration > (state.providerOperationGeneration ?? -1)) invalid('recoveryAttempt'); +} + +function validateResumeGeneration(state: GoalSessionState): void { + const resume = state.resumeIntent; + if (!resume) return; + if (resume.controllerEpoch > state.controllerEpoch + || resume.operationGeneration > (state.providerOperationGeneration ?? -1)) invalid('resumeIntent'); +} + +function validateRecoveryIdentity(state: GoalSessionState): void { + const recovery = state.recoveryAttempt; + if (!recovery) return; + if (state.recoveryAttemptId !== recovery.attemptId) invalid('recoveryAttemptId'); + if ((recovery.authoritativeAttemptId === undefined) !== (recovery.authoritativeExecutionId === undefined)) { + invalid('recovery authoritative identity'); + } + if (hasMismatchedRecoveryTurnIdentity(state)) invalid('recovery authoritative identity'); + if (recovery.sessionStatus !== undefined && recovery.sessionStatus !== state.status) { + invalid('recovery sessionStatus'); + } + if (recovery.authoritativeTurnStatus !== undefined + && recovery.authoritativeTurnStatus !== state.activeTurn?.status) invalid('recovery turnStatus'); +} + +function hasMismatchedRecoveryTurnIdentity(state: GoalSessionState): boolean { + const recovery = state.recoveryAttempt; + if (recovery?.authoritativeAttemptId === undefined) return false; + return recovery.authoritativeAttemptId !== state.activeTurn?.attemptId + || recovery.authoritativeExecutionId !== state.activeTurn?.executionId; +} + +function validateCompletedResume(state: GoalSessionState): void { + const resume = state.resumeIntent; + const completed = state.completedResume; + if (!resume || !completed) return; + if (resume.operationId !== completed.operationId + || resume.operationGeneration !== completed.operationGeneration + || resume.kind !== completed.kind + || resume.controllerEpoch !== completed.controllerEpoch + || resume.phase !== 'settled') invalid('completedResume'); +} + +function validateResumeTurn(state: GoalSessionState): void { + const resume = state.resumeIntent; + if (resume?.kind === 'active_turn' || resume?.kind === 'recovered_after_turn') { + if (!resume.turnId || resume.turnId !== state.activeTurn?.turnId) invalid('resumeIntent.turnId'); + return; + } + if (resume?.turnId !== undefined) invalid('resumeIntent.turnId'); +} + +function validateProviderOpenGeneration(state: GoalSessionState): void { + const generation = state.providerOpenOperationGeneration; + if (generation !== undefined && generation > (state.providerOperationGeneration ?? -1)) { + invalid('providerOpenOperationGeneration'); + } + if ((state.providerOpenAttemptId === undefined) !== (generation === undefined)) invalid('provider open identity'); +} + +function validateActiveTurnGeneration(state: GoalSessionState): void { + const generation = state.activeTurn?.providerOperationGeneration; + if (generation !== undefined && generation > (state.providerOperationGeneration ?? -1)) { + invalid('activeTurn.providerOperationGeneration'); + } +} + +function validateStateCollections(state: GoalSessionState): void { + if (state.completedTurns && (state.completedTurns.some(turn => !state.completedTurnIds.includes(turn.turnId)) + || state.completedTurns.length !== state.completedTurnIds.length)) invalid('completedTurns'); + if (new Set(state.completedTurnIds).size !== state.completedTurnIds.length + || state.completedTurns && new Set(state.completedTurns.map(turn => turn.turnId)).size !== state.completedTurns.length) { + invalid('completedTurns'); + } + if (state.usageAccounting && new Set(state.usageAccounting.occurrences).size !== state.usageAccounting.occurrences.length) { + invalid('usageAccounting.occurrences'); + } + validateCompletedCurrentTurnIdentities(state); +} + +function validateCompletedCurrentTurnIdentities(state: GoalSessionState): void { + for (const completed of state.completedTurns ?? []) { + if (state.activeTurn?.turnId !== completed.turnId) continue; + if (state.activeTurn.executionId !== completed.executionId + || state.activeTurn.attemptId !== completed.attemptId + || !['completed', 'cancelled', 'failed'].includes(state.activeTurn.status)) { + invalid('completed/current turn identity'); + } + } +} + +function validateModelGenerations(state: GoalSessionState): void { + const intents = state.modelChangeIntents ?? []; + validateModelIntentOrder(intents); + const tail = intents.at(-1); + validateModelIntentTail(state, tail); + if ((state.modelChangeGeneration ?? 0) < (tail?.generation ?? 0)) invalid('modelChangeGeneration'); + validatePendingModelChange(state, tail); + const effectiveIntents = intents.length ? intents : state.modelChangeIntent ? [state.modelChangeIntent] : []; + for (const intent of effectiveIntents) validateModelIntentRelationships(intent); + validateEffectiveModel(state, tail); + validateActiveTurnModelChange(state, effectiveIntents); +} + +function validateModelIntentOrder(intents: GoalModelChangeIntent[]): void { + if (new Set(intents.map(intent => intent.modelChangeId)).size !== intents.length) invalid('duplicate modelChangeIds'); + const generations = intents.map(intent => intent.generation ?? 0); + if (generations.some((generation, index) => index > 0 && generation <= generations[index - 1])) { + invalid('modelChangeIntents.generation'); + } +} + +function validateModelIntentTail(state: GoalSessionState, tail: GoalModelChangeIntent | undefined): void { + if (state.modelChangeIntents === undefined) return; + if (state.modelChangeIntent && (!tail || JSON.stringify(state.modelChangeIntent) !== JSON.stringify(tail))) { + invalid('modelChangeIntent tail'); + } + if (!state.modelChangeIntent && tail) invalid('modelChangeIntent tail'); +} + +function validatePendingModelChange(state: GoalSessionState, tail: GoalModelChangeIntent | undefined): void { + if (state.pendingModelChange === undefined) return; + if (!tail || tail.model !== state.pendingModelChange || tail.phase === 'committed' || tail.phase === 'superseded') { + invalid('pendingModelChange'); + } +} + +function validateEffectiveModel(state: GoalSessionState, tail: GoalModelChangeIntent | undefined): void { + if (tail?.phase !== 'committed' || tail.acknowledgement?.effectiveModel === undefined) return; + if (state.pendingModelChange === undefined && state.currentModel !== tail.acknowledgement.effectiveModel) { + invalid('currentModel/model acknowledgement'); + } +} + +function validateActiveTurnModelChange(state: GoalSessionState, intents: GoalModelChangeIntent[]): void { + const active = state.activeTurn?.modelChange; + if (!active) return; + const intent = intents.find(candidate => candidate.modelChangeId === active.modelChangeId); + if (!intent || intent.generation !== active.generation || intent.model !== state.activeTurn?.requestedModel) { + invalid('activeTurn.modelChange'); + } + if (intent.invocationEvidence + && (intent.invocationEvidence.executionId !== state.activeTurn?.executionId + || intent.invocationEvidence.attemptId !== state.activeTurn.attemptId)) { + invalid('activeTurn model invocation evidence'); + } +} + +function validateModelIntentRelationships(intent: GoalModelChangeIntent): void { + const hasLease = hasModelLease(intent); + validateCompleteModelLease(intent, hasLease); + validatePendingModelPhase(intent, hasLease); + validateProviderInDoubtModelPhase(intent, hasLease); + validateSettledModelPhase(intent); + if (intent.acknowledgement?.requestedModel !== undefined + && intent.acknowledgement.requestedModel !== intent.model) invalid('acknowledgement model mismatch'); + validateModelInvocationEvidence(intent, hasLease); + validatePreviousModelInvocationEvidence(intent); +} + +function validatePreviousModelInvocationEvidence(intent: GoalModelChangeIntent): void { + const previous = intent.previousInvocationEvidence; + if (!previous) return; + if (intent.phase !== 'committed' || previous.modelChangeId !== intent.modelChangeId + || previous.generation !== intent.generation || previous.requestedModel !== intent.model + || previous.effectiveModel !== intent.acknowledgement?.effectiveModel + || intent.acknowledgement?.appliesAt !== 'next_turn' + || intent.acknowledgement.outcome !== 'acknowledged') invalid('previous model invocation evidence'); +} + +function hasModelLease(intent: GoalModelChangeIntent): boolean { + return intent.applicationToken !== undefined + || intent.applicationControllerEpoch !== undefined + || intent.leaseExpiresAt !== undefined; +} + +function validateCompleteModelLease(intent: GoalModelChangeIntent, hasLease: boolean): void { + if (hasLease && (!intent.applicationToken || intent.applicationControllerEpoch === undefined || !intent.leaseExpiresAt)) { + invalid('model application lease'); + } +} + +function validatePendingModelPhase(intent: GoalModelChangeIntent, hasLease: boolean): void { + if (intent.phase !== 'pending' && intent.phase !== undefined) return; + if (hasLease || intent.acknowledgement || intent.invocationEvidence) invalid('pending model phase'); +} + +function validateProviderInDoubtModelPhase(intent: GoalModelChangeIntent, hasLease: boolean): void { + if (intent.phase !== 'provider_in_doubt') return; + if (!hasLease || intent.acknowledgement || intent.invocationEvidence) invalid('provider_in_doubt model phase'); +} + +function validateSettledModelPhase(intent: GoalModelChangeIntent): void { + if ((intent.phase === 'committed' || intent.phase === 'superseded') && !intent.acknowledgement) { + invalid('settled model acknowledgement'); + } +} + +function validateModelInvocationEvidence(intent: GoalModelChangeIntent, hasLease: boolean): void { + const evidence = intent.invocationEvidence; + if (!evidence) return; + if (intent.phase !== 'committed' || hasLease + || evidence.modelChangeId !== intent.modelChangeId + || evidence.generation !== intent.generation + || evidence.requestedModel !== intent.model + || evidence.effectiveModel !== intent.acknowledgement?.effectiveModel + || intent.acknowledgement?.appliesAt !== 'next_turn' + || intent.acknowledgement.outcome !== 'acknowledged') invalid('model invocation evidence'); +} + +function invalid(field: string): never { + throw new GoalSessionContractError(`Durable goal session contains an invalid ${field}`, 'INVALID_DURABLE_STATE'); +} diff --git a/packages/core/src/agents/goalSession/durableStateSecurity.ts b/packages/core/src/agents/goalSession/durableStateSecurity.ts new file mode 100644 index 000000000..12820c3ba --- /dev/null +++ b/packages/core/src/agents/goalSession/durableStateSecurity.ts @@ -0,0 +1,375 @@ +import type { + GoalCancellationIntent, + GoalCompletedRecovery, + GoalCompletedResume, + GoalCompletedTurn, + GoalModelChangeAcknowledgement, + GoalModelChangeIntent, + GoalProviderBarrierIntent, + GoalRecoveryAttempt, + GoalRepositoryIdentity, + GoalResumeIntent, + GoalSessionInitializationIntent, + GoalSessionJsonValue, + GoalSessionState, + GoalTurnState, + GoalUsageAccounting, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { validateStateRelationships } from './durableStateRelationships.js'; +import { sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +const SECRET = /(?:Bearer\s*\S+|gh[oprsu]_|github_pat_|sk-|AKIA|secret|token|password|credential|private.?key|-----BEGIN|https?:\/\/[^\s]*@)/i; +const MAX_COMPLETED_TURNS = 10_000; +const MAX_MODEL_INTENTS = 512; +const MAX_USAGE_OCCURRENCES = 256; + +const STATE_FIELDS = [ + 'goalId', 'sessionId', 'provider', 'providerSessionId', 'recoveryMetadata', 'controllerEpoch', 'status', + 'currentModel', 'requestedModel', 'pendingModelChange', 'pendingAfterTurnPause', 'activeTurn', + 'completedTurnIds', 'completedTurns', 'initializationIntent', 'providerOpenAttemptId', + 'providerOpenOperationGeneration', 'retryTurn', 'recoveryAttemptId', 'recoveryAttempt', 'completedRecovery', + 'providerOperationGeneration', 'providerBarrierIntent', 'resumeIntent', 'completedResume', 'cancellationIntent', + 'modelChangeIntent', 'modelChangeIntents', 'modelChangeGeneration', 'usageAccounting', 'failureReason', + 'version', 'createdAt', 'updatedAt', +] as const; + +/** + * Reconstructs a durable state into fresh closed DTOs. No caller receives the + * persistence object's prototype or excess properties, and any malformed known + * field rejects before a supervisor can mutate history, state, events, or a + * provider. This is intentionally not a scrubber: ambiguous durable identity + * is evidence of corruption and must never be repaired by invention. + */ +export function decodeDurableGoalSessionState(value: unknown): GoalSessionState { + const state = record(value, STATE_FIELDS, 'goal session state'); + const result: GoalSessionState = { + goalId: id(state.goalId, 'goalId'), + sessionId: id(state.sessionId, 'sessionId'), + provider: id(state.provider, 'provider'), + controllerEpoch: integer(state.controllerEpoch, 'controllerEpoch'), + status: closed(state.status, ['initializing', 'idle', 'running', 'pause_requested', 'paused', 'cancelling', 'terminated', 'failed'], 'status'), + completedTurnIds: idArray(state.completedTurnIds, 'completedTurnIds', MAX_COMPLETED_TURNS), + version: positiveInteger(state.version, 'version'), + createdAt: timestamp(state.createdAt, 'createdAt'), + updatedAt: timestamp(state.updatedAt, 'updatedAt'), + }; + optionalId(state, result, 'providerSessionId'); + optionalId(state, result, 'currentModel'); + optionalId(state, result, 'requestedModel'); + optionalId(state, result, 'pendingModelChange'); + optionalId(state, result, 'providerOpenAttemptId'); + optionalId(state, result, 'recoveryAttemptId'); + optionalDiagnostic(state, result, 'failureReason'); + optionalInteger(state, result, 'providerOpenOperationGeneration'); + optionalInteger(state, result, 'providerOperationGeneration'); + optionalInteger(state, result, 'modelChangeGeneration'); + if (state.pendingAfterTurnPause !== undefined) result.pendingAfterTurnPause = boolean(state.pendingAfterTurnPause, 'pendingAfterTurnPause'); + if (state.recoveryMetadata !== undefined) { + result.recoveryMetadata = sanitizeRecoveryMetadata(state.recoveryMetadata as GoalSessionJsonValue, result.provider); + } + if (state.activeTurn !== undefined) result.activeTurn = decodeTurn(state.activeTurn); + if (state.completedTurns !== undefined) result.completedTurns = array(state.completedTurns, 'completedTurns', MAX_COMPLETED_TURNS).map(decodeCompletedTurn); + if (state.initializationIntent !== undefined) result.initializationIntent = decodeInitialization(state.initializationIntent); + if (state.retryTurn !== undefined) result.retryTurn = decodeRetryTurn(state.retryTurn); + if (state.recoveryAttempt !== undefined) result.recoveryAttempt = decodeRecovery(state.recoveryAttempt); + if (state.completedRecovery !== undefined) result.completedRecovery = decodeCompletedRecovery(state.completedRecovery); + if (state.providerBarrierIntent !== undefined) result.providerBarrierIntent = decodeBarrier(state.providerBarrierIntent); + if (state.resumeIntent !== undefined) result.resumeIntent = decodeResume(state.resumeIntent); + if (state.completedResume !== undefined) result.completedResume = decodeCompletedResume(state.completedResume); + if (state.cancellationIntent !== undefined) result.cancellationIntent = decodeCancellation(state.cancellationIntent); + if (state.modelChangeIntent !== undefined) result.modelChangeIntent = decodeModelIntent(state.modelChangeIntent); + if (state.modelChangeIntents !== undefined) result.modelChangeIntents = array( + state.modelChangeIntents, 'modelChangeIntents', MAX_MODEL_INTENTS, + ).map(decodeModelIntent); + if (state.usageAccounting !== undefined) result.usageAccounting = decodeUsageAccounting(state.usageAccounting); + validateStateRelationships(result); + return result; +} + +/** @deprecated Strict reopening replaced mutation-based legacy scrubbing. */ +export const stripLegacyStateExtras = decodeDurableGoalSessionState; + +function decodeTurn(value: unknown): GoalTurnState { + const input = record(value, [ + 'turnId', 'executionId', 'attemptId', 'executionEpoch', 'objective', 'requestedModel', 'repository', + 'providerOperationGeneration', 'modelChange', 'status', + ], 'activeTurn'); + const result: GoalTurnState = { + turnId: id(input.turnId, 'activeTurn.turnId'), + executionId: id(input.executionId, 'activeTurn.executionId'), + attemptId: id(input.attemptId, 'activeTurn.attemptId'), + executionEpoch: integer(input.executionEpoch, 'activeTurn.executionEpoch'), + objective: diagnostic(input.objective, 'activeTurn.objective', 2048), + requestedModel: id(input.requestedModel, 'activeTurn.requestedModel'), + repository: decodeRepository(input.repository), + status: closed(input.status, ['running', 'pause_requested', 'paused', 'completed', 'cancelled', 'failed'], 'activeTurn.status'), + }; + if (input.providerOperationGeneration !== undefined) { + result.providerOperationGeneration = integer(input.providerOperationGeneration, 'activeTurn.providerOperationGeneration'); + } + if (input.modelChange !== undefined) { + const modelChange = record(input.modelChange, ['modelChangeId', 'generation', 'previousModel'], 'activeTurn.modelChange'); + result.modelChange = { + modelChangeId: id(modelChange.modelChangeId, 'activeTurn.modelChange.modelChangeId'), + generation: integer(modelChange.generation, 'activeTurn.modelChange.generation'), + previousModel: modelChange.previousModel === undefined + ? undefined : id(modelChange.previousModel, 'activeTurn.modelChange.previousModel'), + }; + } + return result; +} + +function decodeRepository(value: unknown): GoalRepositoryIdentity { + const input = record(value, ['repository', 'worktreePath', 'branch', 'headSha'], 'repository'); + const worktreePath = string(input.worktreePath, 'repository.worktreePath', 4096); + if (!worktreePath.startsWith('/') || worktreePath.includes('\0')) invalid('repository.worktreePath'); + const result: GoalRepositoryIdentity = { + repository: diagnostic(input.repository, 'repository.repository', 1024), + worktreePath, + branch: diagnostic(input.branch, 'repository.branch', 512), + }; + if (input.headSha !== undefined) result.headSha = id(input.headSha, 'repository.headSha'); + return result; +} + +function decodeCompletedTurn(value: unknown): GoalCompletedTurn { + const input = record(value, ['turnId', 'executionId', 'attemptId'], 'completed turn'); + return { turnId: id(input.turnId, 'completedTurn.turnId'), executionId: id(input.executionId, 'completedTurn.executionId'), attemptId: id(input.attemptId, 'completedTurn.attemptId') }; +} + +function decodeInitialization(value: unknown): GoalSessionInitializationIntent { + const input = record(value, ['attemptId', 'deterministicOpenKey', 'recordedAt'], 'initializationIntent'); + return { attemptId: id(input.attemptId, 'initializationIntent.attemptId'), deterministicOpenKey: id(input.deterministicOpenKey, 'initializationIntent.deterministicOpenKey'), recordedAt: timestamp(input.recordedAt, 'initializationIntent.recordedAt') }; +} + +function decodeRetryTurn(value: unknown): NonNullable { + const input = record(value, ['turnId', 'executionId', 'crashedAttemptId'], 'retryTurn'); + return { turnId: id(input.turnId, 'retryTurn.turnId'), executionId: id(input.executionId, 'retryTurn.executionId'), crashedAttemptId: id(input.crashedAttemptId, 'retryTurn.crashedAttemptId') }; +} + +function decodeRecovery(value: unknown): GoalRecoveryAttempt { + const input = record(value, [ + 'operationToken', 'operationGeneration', 'executionId', 'attemptId', 'controllerEpoch', + 'authoritativeAttemptId', 'authoritativeExecutionId', 'sessionStatus', 'authoritativeTurnStatus', + 'claimedAt', 'leaseExpiresAt', 'phase', + ], 'recoveryAttempt'); + const result: GoalRecoveryAttempt = { + operationToken: id(input.operationToken, 'recoveryAttempt.operationToken'), + operationGeneration: integer(input.operationGeneration, 'recoveryAttempt.operationGeneration'), + executionId: id(input.executionId, 'recoveryAttempt.executionId'), + attemptId: id(input.attemptId, 'recoveryAttempt.attemptId'), + controllerEpoch: integer(input.controllerEpoch, 'recoveryAttempt.controllerEpoch'), + claimedAt: timestamp(input.claimedAt, 'recoveryAttempt.claimedAt'), + leaseExpiresAt: timestamp(input.leaseExpiresAt, 'recoveryAttempt.leaseExpiresAt'), + }; + if (input.authoritativeAttemptId !== undefined) result.authoritativeAttemptId = id(input.authoritativeAttemptId, 'recoveryAttempt.authoritativeAttemptId'); + if (input.authoritativeExecutionId !== undefined) result.authoritativeExecutionId = id(input.authoritativeExecutionId, 'recoveryAttempt.authoritativeExecutionId'); + if (input.sessionStatus !== undefined) result.sessionStatus = closed(input.sessionStatus, ['initializing', 'idle', 'running', 'pause_requested', 'paused', 'cancelling', 'terminated', 'failed'], 'recoveryAttempt.sessionStatus') as GoalRecoveryAttempt['sessionStatus']; + if (input.authoritativeTurnStatus !== undefined) result.authoritativeTurnStatus = closed(input.authoritativeTurnStatus, ['running', 'pause_requested', 'paused', 'completed', 'cancelled', 'failed'], 'recoveryAttempt.authoritativeTurnStatus') as GoalRecoveryAttempt['authoritativeTurnStatus']; + if (input.phase !== undefined) result.phase = closed(input.phase, ['claimed', 'provider_in_doubt'], 'recoveryAttempt.phase') as GoalRecoveryAttempt['phase']; + return result; +} + +function decodeCompletedRecovery(value: unknown): GoalCompletedRecovery { + const input = record(value, ['operationToken', 'controllerEpoch', 'outcome', 'reason'], 'completedRecovery'); + return { operationToken: id(input.operationToken, 'completedRecovery.operationToken'), controllerEpoch: integer(input.controllerEpoch, 'completedRecovery.controllerEpoch'), outcome: closed(input.outcome, ['alive', 'resumed', 'failed'], 'completedRecovery.outcome'), reason: diagnostic(input.reason, 'completedRecovery.reason', 512) }; +} + +function decodeBarrier(value: unknown): GoalProviderBarrierIntent { + const input = record(value, ['generation', 'operationId', 'kind', 'phase', 'claimedAt', 'pendingCancellationId'], 'providerBarrierIntent'); + const result: GoalProviderBarrierIntent = { + generation: integer(input.generation, 'providerBarrierIntent.generation'), + operationId: id(input.operationId, 'providerBarrierIntent.operationId'), + kind: closed(input.kind, ['cancellation', 'terminal', 'replacement', 'lease_expiry'], 'providerBarrierIntent.kind'), + phase: closed(input.phase, ['pending', 'published'], 'providerBarrierIntent.phase'), + claimedAt: timestamp(input.claimedAt, 'providerBarrierIntent.claimedAt'), + }; + if (input.pendingCancellationId !== undefined) result.pendingCancellationId = id(input.pendingCancellationId, 'providerBarrierIntent.pendingCancellationId'); + return result; +} + +function decodeResume(value: unknown): GoalResumeIntent { + const input = record(value, ['executionId', 'attemptId', 'operationId', 'operationGeneration', 'kind', 'controllerEpoch', 'turnId', 'claimedAt', 'leaseExpiresAt', 'phase'], 'resumeIntent'); + const result: GoalResumeIntent = { + executionId: id(input.executionId, 'resumeIntent.executionId'), attemptId: id(input.attemptId, 'resumeIntent.attemptId'), + operationId: id(input.operationId, 'resumeIntent.operationId'), operationGeneration: integer(input.operationGeneration, 'resumeIntent.operationGeneration'), + kind: closed(input.kind, ['active_turn', 'after_turn', 'recovered_after_turn'], 'resumeIntent.kind'), + controllerEpoch: integer(input.controllerEpoch, 'resumeIntent.controllerEpoch'), claimedAt: timestamp(input.claimedAt, 'resumeIntent.claimedAt'), + leaseExpiresAt: timestamp(input.leaseExpiresAt, 'resumeIntent.leaseExpiresAt'), phase: closed(input.phase, ['claimed', 'provider_in_doubt', 'settled'], 'resumeIntent.phase'), + }; + if (input.turnId !== undefined) result.turnId = id(input.turnId, 'resumeIntent.turnId'); + return result; +} + +function decodeCompletedResume(value: unknown): GoalCompletedResume { + const input = record(value, ['operationId', 'operationGeneration', 'kind', 'controllerEpoch'], 'completedResume'); + return { operationId: id(input.operationId, 'completedResume.operationId'), operationGeneration: integer(input.operationGeneration, 'completedResume.operationGeneration'), kind: closed(input.kind, ['active_turn', 'after_turn', 'recovered_after_turn'], 'completedResume.kind'), controllerEpoch: integer(input.controllerEpoch, 'completedResume.controllerEpoch') }; +} + +function decodeCancellation(value: unknown): GoalCancellationIntent { + const input = record(value, ['cancellationId', 'reason', 'claimedAt', 'pendingContext'], 'cancellationIntent'); + const result: GoalCancellationIntent = { + cancellationId: id(input.cancellationId, 'cancellationIntent.cancellationId'), + reason: diagnostic(input.reason, 'cancellationIntent.reason', 512), + claimedAt: timestamp(input.claimedAt, 'cancellationIntent.claimedAt'), + }; + if (input.pendingContext !== undefined) { + const pending = record(input.pendingContext, ['initializationIntent', 'activeTurn'], 'cancellationIntent.pendingContext'); + const active = pending.activeTurn === undefined ? undefined : decodeCompletedTurn(pending.activeTurn); + result.pendingContext = { initializationIntent: decodeInitialization(pending.initializationIntent), activeTurn: active }; + } + return result; +} + +function decodeModelIntent(value: unknown): GoalModelChangeIntent { + const input = record(value, [ + 'modelChangeId', 'model', 'requestedAt', 'generation', 'previousModel', 'phase', 'applicationToken', + 'applicationControllerEpoch', 'leaseExpiresAt', 'acknowledgement', 'invocationEvidence', + 'previousInvocationEvidence', + ], 'modelChangeIntent'); + const result: GoalModelChangeIntent = { + modelChangeId: id(input.modelChangeId, 'modelChangeIntent.modelChangeId'), model: id(input.model, 'modelChangeIntent.model'), + requestedAt: timestamp(input.requestedAt, 'modelChangeIntent.requestedAt'), + }; + if (input.generation !== undefined) result.generation = integer(input.generation, 'modelChangeIntent.generation'); + if (input.previousModel !== undefined) result.previousModel = id(input.previousModel, 'modelChangeIntent.previousModel'); + if (input.phase !== undefined) result.phase = closed(input.phase, ['pending', 'provider_in_doubt', 'committed', 'superseded_in_doubt', 'superseded'], 'modelChangeIntent.phase') as GoalModelChangeIntent['phase']; + if (input.applicationToken !== undefined) result.applicationToken = id(input.applicationToken, 'modelChangeIntent.applicationToken'); + if (input.applicationControllerEpoch !== undefined) result.applicationControllerEpoch = integer(input.applicationControllerEpoch, 'modelChangeIntent.applicationControllerEpoch'); + if (input.leaseExpiresAt !== undefined) result.leaseExpiresAt = timestamp(input.leaseExpiresAt, 'modelChangeIntent.leaseExpiresAt'); + if (input.acknowledgement !== undefined) result.acknowledgement = decodeAcknowledgement(input.acknowledgement); + if (input.invocationEvidence !== undefined) { + result.invocationEvidence = decodeInvocationEvidence(input.invocationEvidence, 'invocationEvidence'); + } + if (input.previousInvocationEvidence !== undefined) { + result.previousInvocationEvidence = decodeInvocationEvidence( + input.previousInvocationEvidence, 'previousInvocationEvidence', + ); + } + return result; +} + +function decodeInvocationEvidence(value: unknown, name: string): NonNullable { + const evidence = record(value, [ + 'executionId', 'attemptId', 'modelChangeId', 'generation', 'occurrenceId', + 'requestedModel', 'effectiveModel', 'acceptedAt', + ], `modelChangeIntent.${name}`); + return { + executionId: id(evidence.executionId, `${name}.executionId`), + attemptId: id(evidence.attemptId, `${name}.attemptId`), + modelChangeId: id(evidence.modelChangeId, `${name}.modelChangeId`), + generation: integer(evidence.generation, `${name}.generation`), + occurrenceId: id(evidence.occurrenceId, `${name}.occurrenceId`), + requestedModel: id(evidence.requestedModel, `${name}.requestedModel`), + effectiveModel: id(evidence.effectiveModel, `${name}.effectiveModel`), + acceptedAt: timestamp(evidence.acceptedAt, `${name}.acceptedAt`), + }; +} + +function decodeAcknowledgement(value: unknown): GoalModelChangeAcknowledgement { + const input = record(value, ['outcome', 'requestedModel', 'appliesAt', 'effectiveModel'], 'model acknowledgement'); + const result: GoalModelChangeAcknowledgement = { requestedModel: id(input.requestedModel, 'acknowledgement.requestedModel'), appliesAt: closed(input.appliesAt, ['immediate', 'next_safe_boundary', 'next_turn'], 'acknowledgement.appliesAt') }; + if (input.outcome !== undefined) result.outcome = closed(input.outcome, ['acknowledged', 'outside_retry_horizon'], 'acknowledgement.outcome') as GoalModelChangeAcknowledgement['outcome']; + if (input.effectiveModel !== undefined) result.effectiveModel = id(input.effectiveModel, 'acknowledgement.effectiveModel'); + return result; +} + +function decodeUsageAccounting(value: unknown): GoalUsageAccounting { + const input = record(value, ['version', 'lastWatermark', 'occurrences'], 'usageAccounting'); + if (input.version !== 1) invalid('usageAccounting.version'); + return { version: 1, lastWatermark: integer(input.lastWatermark, 'usageAccounting.lastWatermark'), occurrences: idArray(input.occurrences, 'usageAccounting.occurrences', MAX_USAGE_OCCURRENCES) }; +} + +function record(value: unknown, fields: T, name: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) invalid(name); + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) invalid(name); + const allowed = new Set(fields); + const descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.getOwnPropertySymbols(value).length > 0 + || Object.entries(descriptors).some(([key, descriptor]) => + !allowed.has(key) || !descriptor.enumerable || !('value' in descriptor))) { + invalid(`${name} excess or accessor field`); + } + return value as Record; +} + +function id(value: unknown, name: string): string { + if (!isSafeIdentifier(value)) invalid(name); + return value; +} + +function string(value: unknown, name: string, max: number): string { + if (typeof value !== 'string' || !value || Buffer.byteLength(value) > max || value.includes('\0')) invalid(name); + return value; +} + +function diagnostic(value: unknown, name: string, max: number): string { + const result = string(value, name, max); + if (SECRET.test(result)) invalid(name); + return result; +} + +function integer(value: unknown, name: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) invalid(name); + return value; +} + +function positiveInteger(value: unknown, name: string): number { + const result = integer(value, name); + if (result === 0) invalid(name); + return result; +} + +function boolean(value: unknown, name: string): boolean { + if (typeof value !== 'boolean') invalid(name); + return value; +} + +function timestamp(value: unknown, name: string): string { + if (typeof value !== 'string') invalid(name); + const milliseconds = Date.parse(value); + if (!Number.isFinite(milliseconds) || new Date(milliseconds).toISOString() !== value) invalid(name); + return value; +} + +function closed(value: unknown, allowed: readonly T[], name: string): T { + if (typeof value !== 'string' || !allowed.includes(value as T)) invalid(name); + return value as T; +} + +function array(value: unknown, name: string, max: number): unknown[] { + if (!Array.isArray(value) || value.length > max) invalid(name); + if (Object.getPrototypeOf(value) !== Array.prototype || Object.getOwnPropertySymbols(value).length > 0) invalid(name); + const descriptors = Object.getOwnPropertyDescriptors(value); + for (let index = 0; index < value.length; index += 1) { + const descriptor = descriptors[String(index)]; + if (!descriptor || !descriptor.enumerable || !('value' in descriptor)) invalid(name); + } + if (Object.keys(descriptors).some(key => key !== 'length' && !/^(?:0|[1-9][0-9]*)$/.test(key))) invalid(name); + return value; +} + +function idArray(value: unknown, name: string, max: number): string[] { + const result = array(value, name, max).map(item => id(item, name)); + if (new Set(result).size !== result.length) invalid(name); + return result; +} + +function optionalId(input: Record, output: GoalSessionState, key: K): void { + if (input[key as string] !== undefined) (output as unknown as Record)[key as string] = id(input[key as string], String(key)); +} + +function optionalInteger(input: Record, output: GoalSessionState, key: K): void { + if (input[key as string] !== undefined) (output as unknown as Record)[key as string] = integer(input[key as string], String(key)); +} + +function optionalDiagnostic(input: Record, output: GoalSessionState, key: K): void { + if (input[key as string] !== undefined) (output as unknown as Record)[key as string] = diagnostic(input[key as string], String(key), 512); +} + +function invalid(field: string): never { + throw new GoalSessionContractError(`Durable goal session contains an invalid ${field}`, 'INVALID_DURABLE_STATE'); +} diff --git a/packages/core/src/agents/goalSession/errors.ts b/packages/core/src/agents/goalSession/errors.ts new file mode 100644 index 000000000..51696b621 --- /dev/null +++ b/packages/core/src/agents/goalSession/errors.ts @@ -0,0 +1,44 @@ +/** Typed errors shared by the goal-session runtime. */ + +export class GoalSessionContractError extends Error { + constructor(message: string, readonly code: string) { + super(message); + this.name = 'GoalSessionContractError'; + } +} + +export class StaleGoalSessionFenceError extends GoalSessionContractError { + constructor(message = 'The goal session controller fence is stale') { + super(message, 'STALE_FENCE'); + this.name = 'StaleGoalSessionFenceError'; + } +} + +export class UnsupportedGoalSessionTransitionError extends GoalSessionContractError { + constructor(message: string, code: 'UNSUPPORTED_MODEL_TRANSITION' | 'UNSUPPORTED_PROVIDER_TRANSITION') { + super(message, code); + this.name = 'UnsupportedGoalSessionTransitionError'; + } +} + +export class GoalSessionScopeError extends GoalSessionContractError { + constructor(message = 'The provider session is missing or owned by a different goal') { + super(message, 'SESSION_SCOPE_MISMATCH'); + this.name = 'GoalSessionScopeError'; + } +} + +const PROVIDER_OPEN_IN_DOUBT_ERRORS = new WeakSet(); + +/** Internal Codex transport signal; adapter-supplied lookalikes are untrusted. */ +export function providerOpenInDoubtError(): GoalSessionContractError { + const error = new GoalSessionContractError( + 'Codex thread creation is in doubt; exact identifiers were not persisted', 'PROVIDER_OPEN_IN_DOUBT', + ); + PROVIDER_OPEN_IN_DOUBT_ERRORS.add(error); + return error; +} + +export function isProviderOpenInDoubtError(value: unknown): value is GoalSessionContractError { + return value instanceof GoalSessionContractError && PROVIDER_OPEN_IN_DOUBT_ERRORS.has(value); +} diff --git a/packages/core/src/agents/goalSession/firstTurnIdentity.ts b/packages/core/src/agents/goalSession/firstTurnIdentity.ts new file mode 100644 index 000000000..351e2f213 --- /dev/null +++ b/packages/core/src/agents/goalSession/firstTurnIdentity.ts @@ -0,0 +1,32 @@ +import { createHash } from 'node:crypto'; +import type { + GoalProviderCapabilities, + GoalSessionIdentity, + GoalSessionInitializationIntent, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { nowIso } from './support.js'; + +type FirstTurnCrashPolicy = Extract['firstTurnIdCrashPolicy']; + +export function deterministicOpenKey(identity: GoalSessionIdentity & { provider: string }): string { + return createHash('sha256').update(`${identity.provider}\0${identity.goalId}\0${identity.sessionId}`).digest('hex'); +} + +export function createFirstTurnInitializationIntent( + identity: GoalSessionIdentity & { provider: string }, + attemptId: string, +): GoalSessionInitializationIntent { + return { + attemptId, + deterministicOpenKey: deterministicOpenKey(identity), + recordedAt: nowIso(), + }; +} + +export function firstTurnIdentityFailure(policy: FirstTurnCrashPolicy): GoalSessionContractError { + return new GoalSessionContractError( + `The first provider invocation ended before binding its native session ID (${policy})`, + 'FIRST_TURN_ID_NOT_BOUND', + ); +} diff --git a/packages/core/src/agents/goalSession/goalContainerLayout.ts b/packages/core/src/agents/goalSession/goalContainerLayout.ts new file mode 100644 index 000000000..cc84422cb --- /dev/null +++ b/packages/core/src/agents/goalSession/goalContainerLayout.ts @@ -0,0 +1,143 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import type { SupervisedDockerOutput } from '../../claude/docker/dockerExecutor.js'; +import type { + GoalExecutionIdentity, GoalProviderOperationFence, GoalSessionFence, GoalSessionIdentity, +} from './contract.js'; + +export interface GoalContainerLayout { + executionId: string; + containerName: string; + sessionRoot: string; + providerHome: string; + logPath: string; +} + +/** A read-only credential source kept separate from writable provider state. */ +export interface GoalCredentialMount { + source: string; + target: string; + provider?: 'claude' | 'codex' | 'antigravity'; +} + +/** Adapter-facing view of exact in-memory protocol chunks (never persistence). */ +export interface GoalContainerOutputObserver { + next(output: Readonly): void | 'unsubscribe' | Promise; + complete?(): void | Promise; + error?(error: Error): void | Promise; +} + +export interface StartGoalContainerRequest extends GoalSessionFence, GoalExecutionIdentity { + operationFence: GoalProviderOperationFence; + image: string; + command: string[]; + worktreePath: string; + worktreeFingerprint: string; + providerHomeTarget: string; + environment?: Record; + credentialMounts?: ReadonlyArray; + outputObserver?: GoalContainerOutputObserver; + signal?: AbortSignal; + timeout?: number; + taskId?: string; +} + +/** Eager provider process construction is control-scoped and never invents a turn. */ +export interface StartGoalOpenContainerRequest extends GoalSessionIdentity, GoalExecutionIdentity { + controllerEpoch: number; + deterministicOpenKey: string; + operationFence: GoalProviderOperationFence; + image: string; + command: string[]; + worktreePath: string; + worktreeFingerprint: string; + providerHomeTarget: string; + environment?: Record; + credentialMounts?: ReadonlyArray; + outputObserver?: GoalContainerOutputObserver; + signal?: AbortSignal; + timeout?: number; + taskId?: string; +} + +export interface GoalContainerRetentionPolicy { + succeededMs: number; + cancelledMs: number; + failedMs: number; +} + +/** Host resources explicitly approved for this supervisor instance. */ +export interface GoalContainerIsolationPolicy { + environmentKeys: ReadonlyArray; + worktreePaths: ReadonlyArray; + providerHomeTargets: ReadonlyArray; + credentialMounts?: ReadonlyArray; +} + +export const DEFAULT_GOAL_CONTAINER_RETENTION: GoalContainerRetentionPolicy = { + succeededMs: 24 * 60 * 60 * 1000, + cancelledMs: 24 * 60 * 60 * 1000, + failedMs: 7 * 24 * 60 * 60 * 1000, +}; + +export const GOAL_SCOPE_PATTERN = /^[a-f0-9]{24}$/; + +function opaquePart(value: string, length = 16): string { + return createHash('sha256').update(value).digest('hex').slice(0, length); +} + +function goalScopeFor(request: GoalSessionIdentity): string { + return opaquePart(`${request.goalId}\0${request.sessionId}`, 24); +} + +export function validateAbsolutePath(value: string, name: string): void { + if (!path.isAbsolute(value)) throw new Error(`${name} must be an absolute path`); +} + +/** Rejects characters that Docker would parse as additional mount options. */ +export function validateBindMountPath(value: string, name: string): void { + validateAbsolutePath(value, name); + if (/[,=\n\r\0]/.test(value)) { + throw new Error(`${name} may not contain a comma, '=', or control character that could inject Docker --mount options`); + } +} + +export function buildGoalContainerLayout( + baseDirectory: string, + request: GoalSessionFence & GoalExecutionIdentity, +): GoalContainerLayout { + return buildScopedGoalContainerLayout(baseDirectory, request, request.turnId); +} + +export function buildGoalOpenContainerLayout( + baseDirectory: string, + request: StartGoalOpenContainerRequest, +): GoalContainerLayout { + return buildScopedGoalContainerLayout(baseDirectory, request, `open:${request.deterministicOpenKey}`); +} + +function buildScopedGoalContainerLayout( + baseDirectory: string, + request: GoalSessionIdentity & GoalExecutionIdentity & { controllerEpoch: number }, + operationIdentity: string, +): GoalContainerLayout { + validateBindMountPath(baseDirectory, 'Goal container base directory'); + const goalScope = goalScopeFor(request); + const executionId = [ + goalScope, + `e${request.controllerEpoch}`, + opaquePart(operationIdentity, 10), + opaquePart(request.attemptId, 10), + ].join('-'); + const sessionRoot = path.join(baseDirectory, 'goals', goalScope); + const logDir = path.join(sessionRoot, 'logs'); + const logPath = path.join(logDir, `${executionId}.jsonl`); + if (path.dirname(logPath) !== logDir) throw new Error('Derived goal log path escaped the goal log directory'); + return { + executionId, + containerName: `propr-goal-${executionId}`, + sessionRoot, + providerHome: path.join(sessionRoot, 'provider-home'), + logPath, + }; +} diff --git a/packages/core/src/agents/goalSession/goalSessionOpen.ts b/packages/core/src/agents/goalSession/goalSessionOpen.ts new file mode 100644 index 000000000..f5b032cdb --- /dev/null +++ b/packages/core/src/agents/goalSession/goalSessionOpen.ts @@ -0,0 +1,197 @@ +import type { + GoalProviderDuplexTransport, GoalProviderOpenContext, GoalProviderOperationFence, + GoalRepositoryIdentity, GoalSessionAdapter, GoalSessionIdentity, GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { credentialFreeRepositoryIdentity } from './repositorySecurity.js'; +import { assertSafeProviderIdentifier } from './securityBoundary.js'; +import { SUPERVISED_CODEX_MODEL } from './CodexAppServerOpen.js'; + +export interface OpenGoalSessionRequest extends GoalSessionIdentity { + provider: string; + controllerEpoch: number; + supervisedOpen?: GoalSupervisedOpenPlan; +} + +export interface GoalSupervisedOpenClaim { + executionId: string; + attemptId: string; + deterministicOpenKey: string; + operationGeneration: number; + operationFence: GoalProviderOperationFence; +} + +export interface GoalSupervisedOpenPlan { + readonly repository: GoalRepositoryIdentity; + readonly requestedModel: string; + readonly providerHomeTarget: string; + readonly credentialTargets: readonly string[]; +} + +interface GoalSupervisedOpenPlanInternals { + createTransport(claim: Readonly): Promise; + cancelPending(claim: Readonly): Promise; + transferPending(claim: Readonly): void; +} + +const SUPERVISED_OPEN_PLANS = new WeakMap(); + +/** Issues the only runtime-valid supervised-open plan. */ +export function issueGoalSupervisedOpenPlan( + fields: GoalSupervisedOpenPlan, + internals: GoalSupervisedOpenPlanInternals, +): GoalSupervisedOpenPlan { + const plan = Object.freeze(Object.assign(Object.create(null), { + repository: Object.freeze({ ...fields.repository }), + requestedModel: fields.requestedModel, + providerHomeTarget: fields.providerHomeTarget, + credentialTargets: Object.freeze([...fields.credentialTargets]), + })) as GoalSupervisedOpenPlan; + SUPERVISED_OPEN_PLANS.set(plan, internals); + return plan; +} + +export function supervisedOpenPlanInternals(plan: GoalSupervisedOpenPlan): GoalSupervisedOpenPlanInternals { + const internals = SUPERVISED_OPEN_PLANS.get(plan); + if (!internals) throw new GoalSessionContractError( + 'Supervised open plan was not issued by the production factory', 'UNSAFE_PROVIDER_VALUE', + ); + return internals; +} + +export interface GoalOwnedOpenContext { + context: GoalProviderOpenContext; + cancel(): Promise; + transfer(): void; +} + +export async function createClaimedOpenContext(options: { + adapter: Pick; + plan: GoalSupervisedOpenPlan; + claim: GoalSupervisedOpenClaim; + requireCurrent(): Promise; +}): Promise { + const exactClaim = Object.freeze({ ...options.claim }); + const internals = supervisedOpenPlanInternals(options.plan); + const transport = await internals.createTransport(exactClaim); + try { + const current = await options.requireCurrent(); + if (current.providerOpenAttemptId !== exactClaim.attemptId) { + throw new GoalSessionContractError('Supervised provider transport was cancelled after spawn', 'STALE_FENCE'); + } + const context = await validateClaimedEagerOpenContext(options.adapter, { + ...exactClaim, repository: options.plan.repository, + requestedModel: options.plan.requestedModel, providerHomeTarget: options.plan.providerHomeTarget, + credentialTargets: [...options.plan.credentialTargets], transport, + }); + return { + context, cancel: () => internals.cancelPending(exactClaim), + transfer: () => internals.transferPending(exactClaim), + }; + } catch (error) { + await internals.cancelPending(exactClaim).catch(() => undefined); + throw error; + } +} + +export function createOptionalClaimedOpenContext(options: { + adapter: Pick; + plan?: GoalSupervisedOpenPlan; + executionId: string; attemptId: string; openKey?: string; operationGeneration: number; + operationFence: GoalProviderOperationFence; + requireCurrent(): Promise; +}): Promise { + if (!options.plan) return Promise.resolve(undefined); + if (!options.openKey) throw new GoalSessionContractError( + 'Supervised open claim is missing its durable identity', 'OPEN_ATTEMPT_MISSING', + ); + return createClaimedOpenContext({ + adapter: options.adapter, plan: options.plan, + claim: { + executionId: options.executionId, attemptId: options.attemptId, + deterministicOpenKey: options.openKey, operationGeneration: options.operationGeneration, + operationFence: options.operationFence, + }, + requireCurrent: options.requireCurrent, + }); +} + +export async function validateClaimedEagerOpenContext( + adapter: Pick, + context: GoalProviderOpenContext, +): Promise { + if (adapter.provider !== 'codex' || adapter.capabilities.nativeSessionId !== 'eager') { + throw new GoalSessionContractError( + 'Only eager Codex open accepts a claimed supervised context', 'UNSAFE_PROVIDER_VALUE', + ); + } + assertSafeProviderIdentifier(context.executionId); + assertSafeProviderIdentifier(context.attemptId); + if (context.requestedModel !== SUPERVISED_CODEX_MODEL) throw new GoalSessionContractError( + 'Eager Codex open requires exact gpt-5.6-sol', 'MODEL_ACK_MISMATCH', + ); + validateCredentialTargets(context.credentialTargets, 'Codex credential targets are unsafe'); + const repository = await credentialFreeRepositoryIdentity(context.repository); + if (!isExactRepositoryIdentity(repository, context.repository) + || context.providerHomeTarget !== '/home/node/.codex' + || typeof context.transport.write !== 'function' + || typeof context.transport.closeInput !== 'function' + || typeof context.transport.cancel !== 'function' + || !context.transport.output || typeof context.transport.output[Symbol.asyncIterator] !== 'function' + || !context.transport.completion || typeof context.transport.completion.then !== 'function') { + throw new GoalSessionContractError('Eager Codex open context is unsafe', 'UNSAFE_PROVIDER_VALUE'); + } + return { + executionId: context.executionId, attemptId: context.attemptId, + repository, requestedModel: context.requestedModel, + providerHomeTarget: context.providerHomeTarget, + credentialTargets: [...context.credentialTargets], + deterministicOpenKey: context.deterministicOpenKey, + transport: context.transport, + }; +} + +export async function validateSupervisedOpenPlan( + adapter: Pick, + plan: GoalSupervisedOpenPlan, +): Promise { + supervisedOpenPlanInternals(plan); + if (adapter.provider !== 'codex' || adapter.capabilities.nativeSessionId !== 'eager') { + throw new GoalSessionContractError('Supervised eager open is Codex-only', 'UNSAFE_PROVIDER_VALUE'); + } + if (plan.requestedModel !== SUPERVISED_CODEX_MODEL || plan.providerHomeTarget !== '/home/node/.codex' + || !Object.isFrozen(plan) || Object.getPrototypeOf(plan) !== null) throw new GoalSessionContractError( + 'Supervised Codex open plan is not canonical', 'UNSAFE_PROVIDER_VALUE', + ); + const repository = await credentialFreeRepositoryIdentity(plan.repository); + validateCredentialTargets(plan.credentialTargets, 'Supervised Codex open plan is unsafe'); + if (!isExactRepositoryIdentity(repository, plan.repository)) { + throw new GoalSessionContractError('Supervised Codex open plan is unsafe', 'UNSAFE_PROVIDER_VALUE'); + } +} + +function validateCredentialTargets(value: unknown, message: string): asserts value is string[] { + if (!Array.isArray(value) || value.length > 16 + || value.some(target => typeof target !== 'string' + || !target.startsWith('/home/node/.codex/') || target.includes('\0')) + || new Set(value).size !== value.length) { + throw new GoalSessionContractError(message, 'UNSAFE_PROVIDER_VALUE'); + } +} + +export function durableCodexOpenKey(state: GoalSessionState): string | undefined { + const metadata = state.recoveryMetadata; + if (!metadata || Array.isArray(metadata) || typeof metadata !== 'object') return undefined; + const payload = metadata.payload; + if (!payload || Array.isArray(payload) || typeof payload !== 'object') return undefined; + return typeof payload.openKey === 'string' ? payload.openKey : undefined; +} + +function isExactRepositoryIdentity(canonical: GoalRepositoryIdentity, candidate: GoalRepositoryIdentity): boolean { + const keys = Object.keys(candidate); + return keys.every(key => ['repository', 'worktreePath', 'branch', 'headSha'].includes(key)) + && canonical.repository === candidate.repository + && canonical.worktreePath === candidate.worktreePath + && canonical.branch === candidate.branch + && canonical.headSha === candidate.headSha; +} diff --git a/packages/core/src/agents/goalSession/inMemoryGoalSessionFences.ts b/packages/core/src/agents/goalSession/inMemoryGoalSessionFences.ts new file mode 100644 index 000000000..6d622288e --- /dev/null +++ b/packages/core/src/agents/goalSession/inMemoryGoalSessionFences.ts @@ -0,0 +1,52 @@ +import type { + GoalExecutionIdentity, GoalSessionControlTransition, GoalSessionFence, + GoalSessionState, GoalTerminalCommit, +} from './contract.js'; + +export function matchesLiveMessageFence( + state: GoalSessionState | undefined, + fence: GoalSessionFence, + execution: GoalExecutionIdentity, +): state is GoalSessionState { + return Boolean(state && state.controllerEpoch === fence.controllerEpoch + && state.providerBarrierIntent?.phase !== 'pending' + && !['cancelling', 'terminated', 'failed'].includes(state.status) + && state.activeTurn?.turnId === fence.turnId + && state.activeTurn.executionId === execution.executionId + && state.activeTurn.attemptId === execution.attemptId + && !['completed', 'cancelled', 'failed'].includes(state.activeTurn.status)); +} + +export function matchesTransitionLiveFence( + current: GoalSessionState | undefined, + transition: GoalSessionControlTransition, +): current is GoalSessionState { + if (!current || current.controllerEpoch !== transition.fence.controllerEpoch + || current.providerBarrierIntent?.phase === 'pending' + || current.status === 'cancelling' || current.status === 'terminated' || current.status === 'failed') return false; + if (transition.turnScoped !== true) return true; + if (!('turnId' in transition.fence)) return false; + return current.activeTurn?.turnId === transition.fence.turnId + && current.activeTurn.executionId === transition.execution.executionId + && current.activeTurn.attemptId === transition.execution.attemptId + && current.activeTurn.status !== 'completed' + && current.activeTurn.status !== 'cancelled' + && current.activeTurn.status !== 'failed'; +} + +export function terminalCommitKey(completion: GoalTerminalCommit): string { + return JSON.stringify([ + completion.scope, completion.fence.goalId, completion.fence.sessionId, + completion.fence.controllerEpoch, + completion.scope === 'turn' ? completion.fence.turnId : null, + completion.execution.executionId, completion.execution.attemptId, + ]); +} + +export function transitionCommitKey(transition: GoalSessionControlTransition): string { + return JSON.stringify([ + transition.fence.goalId, transition.fence.sessionId, transition.fence.controllerEpoch, + transition.turnScoped === true && 'turnId' in transition.fence ? transition.fence.turnId : null, + transition.execution.executionId, transition.execution.attemptId, transition.transitionId, + ]); +} diff --git a/packages/core/src/agents/goalSession/index.ts b/packages/core/src/agents/goalSession/index.ts new file mode 100644 index 000000000..c6625bf67 --- /dev/null +++ b/packages/core/src/agents/goalSession/index.ts @@ -0,0 +1,60 @@ +export * from './contract.js'; +export { + EAGER_ACTIVE_TURN_PROVIDER_CAPABILITIES, + FIRST_TURN_BOUNDARY_PROVIDER_CAPABILITIES, +} from './providerCapabilities.js'; +export { + GoalSessionContractError, + GoalSessionSupervisor, + StaleGoalSessionFenceError, + UnsupportedGoalSessionTransitionError, + assertCredentialFreeRecoveryMetadata, + firstPendingCorrectiveMessage, +} from './GoalSessionSupervisor.js'; +export type { + GoalSupervisedOpenClaim, + GoalSupervisedOpenPlan, + OpenGoalSessionRequest, + ReconcileGoalSessionResult, + RunGoalTurnRequest, + RunGoalTurnResult, +} from './GoalSessionSupervisor.js'; +export { GoalSessionScopeError } from './errors.js'; +export { AuthoritativeGoalSessionRuntimePorts } from './AuthoritativeGoalSessionRuntimePorts.js'; +export { + createSqliteGoalSessionRuntimePorts, + SqliteGoalSessionControlDomain, +} from './SqliteGoalSessionControlDomain.js'; +export { + DEFAULT_GOAL_CONTAINER_RETENTION, + GoalContainerSupervisor, + buildGoalContainerLayout, + buildGoalOpenContainerLayout, +} from './GoalContainerSupervisor.js'; +export type { + GoalContainerLayout, + GoalContainerIsolationPolicy, + GoalContainerSupervisorOptions, + GoalContainerRetentionPolicy, + GoalContainerOutputObserver, + GoalCredentialMount, + StartGoalContainerRequest, + StartGoalOpenContainerRequest, +} from './GoalContainerSupervisor.js'; +export { DockerGoalSessionRecovery } from './DockerGoalSessionRecovery.js'; +export { + fingerprintGoalWorktree, + normalizeGitRepositoryIdentity, + normalizeGoalRepositoryIdentity, + normalizeCanonicalGoalRepositoryIdentity, +} from './worktreeIdentity.js'; +export { MODEL_CHANGE_SETTLED_RETRY_HORIZON } from './modelChangeProtocol.js'; +export { GOAL_RECOVERY_METADATA_CODEC_VERSION, sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +export { openSupervisedCodexAppServer, SUPERVISED_CODEX_MODEL } from './CodexAppServerOpen.js'; +export { createProviderProtocolDuplex } from './providerProtocolDuplex.js'; +export { createSupervisedCodexAppServerFactory } from './supervisedCodexOpenFactory.js'; +export type { + GoalProviderOpenFactory, SupervisedCodexAppServerFactoryOptions, +} from './supervisedCodexOpenFactory.js'; +export type { GoalRecoveryMetadataV1 } from './recoveryMetadata.js'; +export { decodeDurableGoalSessionState } from './durableStateSecurity.js'; diff --git a/packages/core/src/agents/goalSession/modelApplicationLease.ts b/packages/core/src/agents/goalSession/modelApplicationLease.ts new file mode 100644 index 000000000..bb8898b2d --- /dev/null +++ b/packages/core/src/agents/goalSession/modelApplicationLease.ts @@ -0,0 +1,16 @@ +import type { GoalModelChangeIntent, GoalSessionState } from './contract.js'; + +const MODEL_APPLICATION_LEASE_MS = 30_000; + +export function claimModelApplicationIntent( + intent: GoalModelChangeIntent, + state: GoalSessionState, +): GoalModelChangeIntent { + return { + ...intent, + phase: intent.phase === 'committed' ? 'committed' : 'provider_in_doubt', + applicationToken: `${intent.modelChangeId}:e${state.controllerEpoch}:v${state.version}`, + applicationControllerEpoch: state.controllerEpoch, + leaseExpiresAt: new Date(Date.now() + MODEL_APPLICATION_LEASE_MS).toISOString(), + }; +} diff --git a/packages/core/src/agents/goalSession/modelChangeHistory.ts b/packages/core/src/agents/goalSession/modelChangeHistory.ts new file mode 100644 index 000000000..f8551ec48 --- /dev/null +++ b/packages/core/src/agents/goalSession/modelChangeHistory.ts @@ -0,0 +1,24 @@ +import type { + GoalModelChangeAcknowledgement, GoalModelChangeHistoryPort, + GoalModelChangeRequest, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; + +export async function resolveModelChangeHistory( + historyPort: GoalModelChangeHistoryPort, + request: GoalModelChangeRequest, + options: { operationId: string; appliesAt: 'next_turn' | 'next_safe_boundary'; retainedIntent: boolean }, +): Promise { + const { operationId, appliesAt, retainedIntent } = options; + const history = await historyPort.claim(request, operationId, request.model); + if (history.model !== request.model) { + throw new GoalSessionContractError( + 'Model operationId was already used for a different model', 'MODEL_OPERATION_CONFLICT', + ); + } + if (history.status === 'retired') { + return { outcome: 'outside_retry_horizon', requestedModel: request.model, appliesAt }; + } + return history.status === 'settled' && history.acknowledgement && (!retainedIntent || appliesAt === 'next_turn') + ? history.acknowledgement : undefined; +} diff --git a/packages/core/src/agents/goalSession/modelChangeProtocol.ts b/packages/core/src/agents/goalSession/modelChangeProtocol.ts new file mode 100644 index 000000000..4031c2557 --- /dev/null +++ b/packages/core/src/agents/goalSession/modelChangeProtocol.ts @@ -0,0 +1,158 @@ +import type { + GoalModelChangeAcknowledgement, GoalModelChangeIntent, GoalModelChangeRequest, GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +/** + * Settled generations kept for ambiguous controller retries. Provider-side + * idempotency identities older than this window have no live local caller and + * their ordered audit evidence remains in the append-only event stream. + */ +export const MODEL_CHANGE_SETTLED_RETRY_HORIZON = 64; + +function isSettled(intent: GoalModelChangeIntent): boolean { + return (intent.phase === 'committed' || intent.phase === 'superseded') && !intent.applicationToken; +} + +/** + * Deterministically bounds settled intent history without ever removing an + * unresolved provider obligation. The newest generation is retained even for + * legacy records whose phase was omitted. + */ +export function compactImmediateModelIntents( + intents: readonly GoalModelChangeIntent[], +): GoalModelChangeIntent[] { + if (intents.length <= MODEL_CHANGE_SETTLED_RETRY_HORIZON) return [...intents]; + const settledToRetain = new Set( + intents + .filter(isSettled) + .slice(-MODEL_CHANGE_SETTLED_RETRY_HORIZON) + .map(intent => intent.modelChangeId), + ); + const latestId = intents.at(-1)?.modelChangeId; + return intents.filter(intent => + !isSettled(intent) || settledToRetain.has(intent.modelChangeId) || intent.modelChangeId === latestId); +} + +export function requestedImmediateModelIntent( + state: GoalSessionState, + request: GoalModelChangeRequest, +): { intent?: GoalModelChangeIntent } { + if (request.operationId !== undefined && !isSafeIdentifier(request.operationId)) { + throw new GoalSessionContractError('Model change operationId is invalid', 'INVALID_MODEL_OPERATION_ID'); + } + let intent = request.operationId + ? immediateModelIntents(state).find(value => value.modelChangeId === request.operationId) + : latestImmediateModelIntent(state); + if (!request.operationId && intent?.model !== request.model) intent = undefined; + if (intent && intent.model !== request.model) { + throw new GoalSessionContractError('Model operationId was already used for a different model', 'MODEL_OPERATION_CONFLICT'); + } + return { intent }; +} + +export function assertModelControllable(state: GoalSessionState): void { + if (state.status === 'cancelling' || state.status === 'terminated' || state.status === 'failed') { + throw new GoalSessionContractError( + `Cannot apply a model while the session is ${state.status}`, 'SESSION_NOT_CONTROLLABLE', + ); + } +} + +export function validateImmediateModelAcknowledgement( + request: GoalModelChangeRequest, + state: GoalSessionState, + acknowledgement: GoalModelChangeAcknowledgement, +): void { + if (acknowledgement.requestedModel !== request.model) { + throw new GoalSessionContractError('Provider acknowledged a different requested model', 'MODEL_ACK_MISMATCH'); + } + if (acknowledgement.appliesAt === 'next_turn') { + throw new GoalSessionContractError('Provider deferred beyond its declared model boundary', 'CAPABILITY_ACK_MISMATCH'); + } + if (acknowledgement.appliesAt === 'immediate' + && (state.status === 'running' || state.status === 'pause_requested')) { + throw new GoalSessionContractError( + 'Provider applied a model change before an active-turn safe boundary', 'CAPABILITY_ACK_MISMATCH', + ); + } +} + +export function immediateModelIntents(state: GoalSessionState): GoalModelChangeIntent[] { + if (state.modelChangeIntents?.length) return state.modelChangeIntents; + return state.modelChangeIntent ? [state.modelChangeIntent] : []; +} + +export function latestImmediateModelIntent(state: GoalSessionState): GoalModelChangeIntent | undefined { + return immediateModelIntents(state).at(-1); +} + +export function nextModelGeneration(state: GoalSessionState): number { + const durableMaximum = immediateModelIntents(state).reduce( + (maximum, intent) => Math.max(maximum, intent.generation ?? 0), + 0, + ); + return Math.max(state.modelChangeGeneration ?? 0, durableMaximum) + 1; +} + +export function replaceImmediateModelIntent( + state: GoalSessionState, + replacement: GoalModelChangeIntent, +): GoalModelChangeIntent[] { + return compactImmediateModelIntents(immediateModelIntents(state).map(intent => + intent.modelChangeId === replacement.modelChangeId ? replacement : intent)); +} + +/** Moves exact evidence out of the active slot before a fresh recovery attempt is authoritative. */ +export function prepareModelEvidenceForRecoveredAttempt( + state: GoalSessionState, + execution: { executionId: string; attemptId: string }, +): { modelChangeIntents?: GoalModelChangeIntent[]; modelChangeIntent?: GoalModelChangeIntent } { + const active = state.activeTurn?.modelChange; + if (!active) return {}; + const intent = immediateModelIntents(state).find(candidate => candidate.modelChangeId === active.modelChangeId); + const evidence = intent?.invocationEvidence; + if (!intent || !evidence || evidence.executionId === execution.executionId && evidence.attemptId === execution.attemptId) { + return {}; + } + const replacement = { + ...intent, invocationEvidence: undefined, previousInvocationEvidence: evidence, + }; + const intents = replaceImmediateModelIntent(state, replacement); + return { modelChangeIntents: intents, modelChangeIntent: intents.at(-1) }; +} + +export function hasUnresolvedImmediateModelIntent(state: GoalSessionState): boolean { + return immediateModelIntents(state).some(intent => + Boolean(intent.applicationToken) || (intent.phase !== 'committed' && intent.phase !== 'superseded')); +} + +export function isLiveModelLease(intent: GoalModelChangeIntent, controllerEpoch: number): boolean { + return Boolean(intent.applicationToken + && intent.applicationControllerEpoch === controllerEpoch + && intent.leaseExpiresAt + && Date.parse(intent.leaseExpiresAt) > Date.now()); +} + +export function obsoleteModelIntents( + state: GoalSessionState, + latestModelChangeId: string, + reconciled: boolean, +): { changed: boolean; intents: GoalModelChangeIntent[] } { + let changed = false; + const intents = compactImmediateModelIntents(immediateModelIntents(state).map(intent => { + if (intent.modelChangeId === latestModelChangeId + || intent.phase === 'committed' || intent.phase === 'superseded') return intent; + changed = true; + return { + ...intent, + phase: reconciled ? 'superseded' as const : 'superseded_in_doubt' as const, + acknowledgement: reconciled ? intent.acknowledgement ?? { + requestedModel: intent.model, + appliesAt: 'next_safe_boundary' as const, + } : intent.acknowledgement, + }; + })); + return { changed, intents }; +} diff --git a/packages/core/src/agents/goalSession/pendingOpenOwnership.ts b/packages/core/src/agents/goalSession/pendingOpenOwnership.ts new file mode 100644 index 000000000..92293cf0b --- /dev/null +++ b/packages/core/src/agents/goalSession/pendingOpenOwnership.ts @@ -0,0 +1,79 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import type { SupervisedDockerExecution } from '../../claude/docker/dockerExecutor.js'; +import type { GoalSupervisedOpenClaim } from './goalSessionOpen.js'; + +const execFileAsync = promisify(execFile); +interface PendingOpenIdentity { + goalId: string; + sessionId: string; + attemptId: string; + deterministicOpenKey?: string; +} + +function identity(claim: Readonly): PendingOpenIdentity { + return { + goalId: claim.operationFence.goalId, sessionId: claim.operationFence.sessionId, + attemptId: claim.attemptId, deterministicOpenKey: claim.deterministicOpenKey, + }; +} + +function key(value: PendingOpenIdentity): string { + return `${value.goalId}\0${value.sessionId}\0${value.attemptId}`; +} + +export class PendingOpenOwnership { + private readonly executions = new Map(); + + constructor(private readonly dockerPath = '/usr/bin/docker') {} + + register(claim: Readonly, execution: SupervisedDockerExecution): void { + this.executions.set(key(identity(claim)), execution); + } + + transfer(claim: Readonly): void { this.executions.delete(key(identity(claim))); } + + async cancel(claim: Readonly): Promise { + await this.cancelIdentity(identity(claim)); + } + + async cancelIdentity(value: PendingOpenIdentity): Promise { + const execution = this.executions.get(key(value)); + if (execution) { + this.executions.delete(key(value)); + await execution.cancel(new Error('Pending eager-open ownership was cancelled')); + return; + } + await this.cancelByDurableLabels(value); + } + + private async cancelByDurableLabels(value: PendingOpenIdentity): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < 12; attempt += 1) { + try { + const { stdout } = await execFileAsync(this.dockerPath, [ + 'ps', '-aq', '--filter', `label=propr.goal.id=${value.goalId}`, + '--filter', `label=propr.goal.session=${value.sessionId}`, + '--filter', 'label=propr.goal.scope=open', + '--filter', `label=propr.goal.attempt=${value.attemptId}`, + ...(value.deterministicOpenKey + ? ['--filter', `label=propr.goal.open-key=${value.deterministicOpenKey}`] : []), + ], { timeout: 2_000, maxBuffer: 64 * 1024 }); + const containers = stdout.split('\n').map(item => item.trim()).filter(Boolean); + for (const container of containers) { + await execFileAsync(this.dockerPath, ['rm', '-f', container], { timeout: 4_000, maxBuffer: 64 * 1024 }); + } + if (containers.length > 0 || attempt === 11) return; + await wait(50); + } catch (error) { + lastError = error; + await wait(50); + } + } + throw new Error('Pending eager-open container cleanup failed safely', { cause: lastError }); + } +} + +function wait(milliseconds: number): Promise { + return new Promise(resolve => setTimeout(resolve, milliseconds)); +} diff --git a/packages/core/src/agents/goalSession/providerBarrierProtocol.ts b/packages/core/src/agents/goalSession/providerBarrierProtocol.ts new file mode 100644 index 000000000..1819042c5 --- /dev/null +++ b/packages/core/src/agents/goalSession/providerBarrierProtocol.ts @@ -0,0 +1,83 @@ +import type { + GoalSessionControlFence, GoalSessionRuntimePorts, GoalSessionState, +} from './contract.js'; +import { nextState } from './support.js'; + +const PROVIDER_BOUNDARY_TIMEOUT_MS = 1_000; + +/** Bounds an adapter barrier without leaving a late rejection unobserved. */ +export async function boundedProviderBoundary(operation: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('Provider boundary timed out')), PROVIDER_BOUNDARY_TIMEOUT_MS); + }); + try { + return await Promise.race([operation, timeout]); + } finally { + if (timer) clearTimeout(timer); + void operation.catch(() => undefined); + } +} + +/** Stages lease expiry durably, publishes it, then records replay completion. */ +export async function expireResumeLease(options: { + ports: GoalSessionRuntimePorts; + fence: GoalSessionControlFence; + operationId: string; + operationGeneration: number; + load: () => Promise; + publish: (state: GoalSessionState, generation: number) => Promise; +}): Promise { + const state = await options.load(); + const intent = state.resumeIntent; + if (!intent || intent.operationId !== options.operationId + || intent.operationGeneration !== options.operationGeneration + || state.providerBarrierIntent?.phase === 'pending') return; + const generation = (state.providerOperationGeneration ?? 0) + 1; + const barrierOperationId = `${options.operationId}:lease-expiry`; + const staged = await options.ports.state.compareAndSet(state, nextState(state, { + providerOperationGeneration: generation, + resumeIntent: { ...intent, leaseExpiresAt: new Date(0).toISOString() }, + providerBarrierIntent: { + generation, operationId: barrierOperationId, kind: 'lease_expiry', phase: 'pending', + claimedAt: new Date().toISOString(), + }, + })); + if (!staged) return; + await options.publish(staged, generation); + const current = await options.load(); + if (current.providerBarrierIntent?.operationId !== barrierOperationId) return; + await options.ports.state.compareAndSet(current, nextState(current, { + providerBarrierIntent: { ...current.providerBarrierIntent, phase: 'published' }, + })); +} + +export async function expireRecoveryLease(options: { + ports: GoalSessionRuntimePorts; + fence: GoalSessionControlFence; + operationToken: string; + load: () => Promise; + publish: (generation: number) => Promise; +}): Promise { + const state = await options.load(); + const recovery = state.recoveryAttempt; + if (!recovery || recovery.operationToken !== options.operationToken + || state.providerBarrierIntent?.phase === 'pending') return; + const generation = (state.providerOperationGeneration ?? 0) + 1; + const operationId = `${options.operationToken}:lease-expiry`; + const staged = await options.ports.state.compareAndSet(state, nextState(state, { + providerOperationGeneration: generation, + recoveryAttempt: { ...recovery, leaseExpiresAt: new Date(0).toISOString() }, + providerBarrierIntent: { + generation, operationId, kind: 'lease_expiry', phase: 'pending', + claimedAt: new Date().toISOString(), + }, + })); + if (!staged) return; + await options.publish(generation); + const current = await options.load(); + if (current.providerBarrierIntent?.operationId !== operationId) return; + await options.ports.state.compareAndSet(current, nextState(current, { + providerBarrierIntent: { ...current.providerBarrierIntent, phase: 'published' }, + })); +} diff --git a/packages/core/src/agents/goalSession/providerCapabilities.ts b/packages/core/src/agents/goalSession/providerCapabilities.ts new file mode 100644 index 000000000..89aaf943f --- /dev/null +++ b/packages/core/src/agents/goalSession/providerCapabilities.ts @@ -0,0 +1,34 @@ +export type GoalNativeSessionIdTiming = 'eager' | 'first_turn'; +export type GoalSteeringBoundary = 'active_turn' | 'next_turn'; +export type GoalPauseBoundary = 'active_turn' | 'after_turn'; +export type GoalModelChangeBoundary = 'next_safe_boundary' | 'next_turn'; + +export type GoalProviderCapabilities = { + nativeSessionId: 'eager'; + steering: GoalSteeringBoundary; + pause: GoalPauseBoundary; + modelChange: GoalModelChangeBoundary; +} | { + nativeSessionId: 'first_turn'; + firstTurnIdCrashPolicy: 'retry_deterministically' | 'fail'; + steering: GoalSteeringBoundary; + pause: GoalPauseBoundary; + modelChange: GoalModelChangeBoundary; +}; + +/** Contract fixture for providers with a native active-turn control channel. */ +export const EAGER_ACTIVE_TURN_PROVIDER_CAPABILITIES = { + nativeSessionId: 'eager', + steering: 'active_turn', + pause: 'active_turn', + modelChange: 'next_safe_boundary', +} as const satisfies GoalProviderCapabilities; + +/** Contract fixture for discrete CLIs whose native identity arrives on turn one. */ +export const FIRST_TURN_BOUNDARY_PROVIDER_CAPABILITIES = { + nativeSessionId: 'first_turn', + firstTurnIdCrashPolicy: 'fail', + steering: 'next_turn', + pause: 'after_turn', + modelChange: 'next_turn', +} as const satisfies GoalProviderCapabilities; diff --git a/packages/core/src/agents/goalSession/providerEffectProtocol.ts b/packages/core/src/agents/goalSession/providerEffectProtocol.ts new file mode 100644 index 000000000..ff394a6c5 --- /dev/null +++ b/packages/core/src/agents/goalSession/providerEffectProtocol.ts @@ -0,0 +1,174 @@ +import type { + GoalProviderFirstEffectPort, GoalProviderOperationFence, GoalProviderResumeRequest, + GoalResumeIntent, GoalSessionAdapter, GoalSessionControlFence, GoalSessionState, GoalStartedProviderEffect, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { persistedSnapshot } from './support.js'; +import { assertGoalProviderEffectStage, assertGoalProviderOperationFence } from './providerOperationBoundary.js'; + +const STARTED_PROVIDER_EFFECTS = new WeakSet(); + +type OperationIdentity = Pick + & Partial>; + +export function createProviderOperationFence( + identity: GoalSessionControlFence, + generation: number, + operation: OperationIdentity, +): GoalProviderOperationFence { + const fence = { + goalId: identity.goalId, sessionId: identity.sessionId, + controllerEpoch: identity.controllerEpoch, generation, + kind: operation.kind, operationId: operation.operationId, + leaseExpiresAt: operation.leaseExpiresAt, turnId: operation.turnId, + executionId: operation.executionId, attemptId: operation.attemptId, + }; + assertGoalProviderOperationFence(fence); + return fence; +} + +export function createProviderResumeRequest( + fence: GoalSessionControlFence, + intent: GoalResumeIntent, +): GoalProviderResumeRequest { + return { + goalId: fence.goalId, sessionId: fence.sessionId, controllerEpoch: fence.controllerEpoch, + operationId: intent.operationId, operationGeneration: intent.operationGeneration, + operationPhase: intent.phase === 'settled' ? 'settled' : 'provider_in_doubt', kind: intent.kind, + operationLeaseExpiresAt: intent.leaseExpiresAt, + operationFence: createProviderOperationFence(fence, intent.operationGeneration, { + kind: 'resume', operationId: intent.operationId, leaseExpiresAt: intent.leaseExpiresAt, + turnId: intent.turnId, executionId: intent.executionId, attemptId: intent.attemptId, + }), + }; +} + +/** Builds the only value accepted from a synchronous first-effect callback. */ +export function startedProviderEffect( + completion: Promise, + rollbackOrCancel: () => void | Promise, +): GoalStartedProviderEffect { + if (!isExactNativePromise(completion) || typeof rollbackOrCancel !== 'function') { + throw new GoalSessionContractError( + 'Provider first effect must expose native completion and cleanup ownership', 'INVALID_FIRST_EFFECT_HANDLE', + ); + } + // Observe rejection at ownership acceptance, before a receipt/COMMIT failure + // can divert control into cleanup. The original promise remains unchanged + // and still rejects for its normal consumer. + void completion.catch(() => undefined); + const cleanup = Object.freeze({ kind: 'rollback_or_cancel' as const, run: rollbackOrCancel }); + const handle = Object.create(null) as GoalStartedProviderEffect; + Object.defineProperties(handle, { + completion: { value: completion, enumerable: true }, + cleanup: { value: cleanup, enumerable: true }, + }); + STARTED_PROVIDER_EFFECTS.add(handle); + return Object.freeze(handle); +} + +/** Runtime guard for untyped embedders and JavaScript callers. */ +export function assertStartedProviderEffect(value: unknown): asserts value is GoalStartedProviderEffect { + if (!isExactStartedProviderEffect(value)) { + throw new GoalSessionContractError( + 'Provider first-effect callback must synchronously return a started-effect handle', + 'ASYNC_FIRST_EFFECT_CALLBACK', + ); + } +} + +export async function cleanupStartedProviderEffect(value: GoalStartedProviderEffect): Promise { + await value.cleanup.run(); +} + +export function startedProviderEffectCleanup(value: unknown): GoalStartedProviderEffect['cleanup'] | undefined { + if (!value || typeof value !== 'object') return undefined; + try { + const descriptor = Object.getOwnPropertyDescriptor(value, 'cleanup'); + return descriptor && !('get' in descriptor) && isExactCleanup(descriptor.value) + ? descriptor.value as GoalStartedProviderEffect['cleanup'] : undefined; + } catch { return undefined; } +} + +export async function rollbackStartedProviderPrimitive( + adapter: GoalSessionAdapter, + fence: GoalProviderOperationFence, + state: GoalSessionState, +): Promise { + const request = { + goalId: fence.goalId, sessionId: fence.sessionId, controllerEpoch: fence.controllerEpoch, + reason: 'Authoritative provider-effect transaction failed after start', + cancellationId: fence.operationId, operationGeneration: fence.generation, + operationFence: { ...fence, kind: 'cancel' as const }, + }; + if (!state.providerSessionId && state.initializationIntent && adapter.cancelPending) { + await adapter.cancelPending(request, { + initializationIntent: state.initializationIntent, + activeTurn: state.activeTurn ? { + turnId: state.activeTurn.turnId, + executionId: state.activeTurn.executionId, + attemptId: state.activeTurn.attemptId, + } : undefined, + }); + return; + } + await adapter.cancel(request, persistedSnapshot(state)); +} + +function isExactStartedProviderEffect(value: unknown): value is GoalStartedProviderEffect { + if (!value || typeof value !== 'object' || Object.getPrototypeOf(value) !== null || !Object.isFrozen(value)) return false; + const descriptors = Object.getOwnPropertyDescriptors(value); + const names = Object.keys(descriptors); + const symbols = Object.getOwnPropertySymbols(value); + if (names.length !== 2 || !names.includes('completion') || !names.includes('cleanup') + || symbols.length !== 0 || !STARTED_PROVIDER_EFFECTS.has(value)) return false; + const completion = descriptors.completion; + const cleanup = descriptors.cleanup; + if (!completion || 'get' in completion || !isExactNativePromise(completion.value) + || !cleanup || 'get' in cleanup || !isExactCleanup(cleanup.value)) return false; + return !Object.hasOwn(value, 'then'); +} + +function isExactCleanup(value: unknown): boolean { + if (!value || typeof value !== 'object' || !Object.isFrozen(value)) return false; + const descriptors = Object.getOwnPropertyDescriptors(value); + return Object.getPrototypeOf(value) === Object.prototype + && Object.keys(descriptors).length === 2 + && descriptors.kind?.value === 'rollback_or_cancel' + && typeof descriptors.run?.value === 'function' + && !('get' in descriptors.kind) && !('get' in descriptors.run); +} + +function isExactNativePromise(value: unknown): value is Promise { + return value instanceof Promise && Object.getPrototypeOf(value) === Promise.prototype + && !Object.hasOwn(value, 'then'); +} + +/** Defers an async iterable's real first effect to its first `next()` call. */ +export function providerFirstEffectStream( + port: GoalProviderFirstEffectPort, + fence: GoalProviderOperationFence, + create: () => AsyncIterable, + rebuild: (value: IteratorResult) => IteratorResult, +): AsyncIterable { + assertGoalProviderEffectStage('stream_first_next'); + let iterator: AsyncIterator | undefined; + let started = false; + return { + [Symbol.asyncIterator]: () => ({ + next: async () => { + if (started) return iterator!.next(); + started = true; + return port.start(fence, 'stream_first_next', () => { + iterator = create()[Symbol.asyncIterator](); + if (!iterator.return) throw new GoalSessionContractError( + 'Provider stream must expose synchronous cancellation ownership', 'INVALID_FIRST_EFFECT_HANDLE', + ); + const completion = iterator.next(); + return startedProviderEffect(completion, async () => { await iterator!.return!(); }); + }, rebuild); + }, + return: async () => iterator?.return ? iterator.return() : { done: true, value: undefined }, + }), + }; +} diff --git a/packages/core/src/agents/goalSession/providerFirstEffect.ts b/packages/core/src/agents/goalSession/providerFirstEffect.ts new file mode 100644 index 000000000..68c754b5b --- /dev/null +++ b/packages/core/src/agents/goalSession/providerFirstEffect.ts @@ -0,0 +1,132 @@ +import type { + GoalModelChangeIntent, GoalProviderOperationFence, GoalSessionState, +} from './contract.js'; +import { StaleGoalSessionFenceError } from './errors.js'; +import { compositeOperationId, controlOperationId } from './controlOperationIdentity.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; +import { assertGoalProviderOperationFence } from './providerOperationBoundary.js'; + +/** Validates the complete serializable provider fence against one locked state row. */ +export function assertProviderFirstEffectState( + state: GoalSessionState | null, + fence: GoalProviderOperationFence, +): asserts state is GoalSessionState { + assertGoalProviderOperationFence(fence); + if (!state || state.goalId !== fence.goalId || state.sessionId !== fence.sessionId + || state.controllerEpoch !== fence.controllerEpoch + || (state.providerOperationGeneration ?? 0) !== fence.generation + || state.providerBarrierIntent?.phase === 'pending' + || expired(fence.leaseExpiresAt)) stale(); + assertExactTurn(state, fence); + assertKindAuthority(state, fence); +} + +function assertExactTurn(state: GoalSessionState, fence: GoalProviderOperationFence): void { + if (fence.turnId === undefined) return; + const turn = state.activeTurn; + if (!turn || turn.turnId !== fence.turnId || turn.executionId !== fence.executionId + || turn.attemptId !== fence.attemptId + || (fence.kind === 'turn' || fence.kind === 'steer') + && (turn.providerOperationGeneration ?? 0) !== fence.generation) stale(); +} + +function assertKindAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + switch (fence.kind) { + case 'open': + assertOpenAuthority(state, fence); + return; + case 'turn': + case 'steer': + assertTurnAuthority(state, fence); + return; + case 'pause': + assertPauseAuthority(state, fence); + return; + case 'resume': + assertResumeAuthority(state, fence); + return; + case 'reconcile': + assertRecoveryAuthority(state, fence); + return; + case 'model': + assertModelAuthority(state, fence); + return; + case 'cancel': + assertCancelAuthority(state, fence); + return; + default: + stale(); + } +} + +function assertOpenAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + if (state.providerOpenAttemptId !== fence.operationId + || (fence.attemptId !== undefined && state.providerOpenAttemptId !== fence.attemptId) + || state.providerOpenOperationGeneration !== fence.generation + || state.status === 'cancelling' || terminal(state)) stale(); +} + +function assertTurnAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + const turn = state.activeTurn; + const expectedTurnOperation = turn + ? compositeOperationId('turn', turn.turnId, turn.executionId, turn.attemptId) : undefined; + if (!turn || !['running', 'pause_requested', 'paused'].includes(state.status) + || fence.kind === 'turn' && fence.operationId !== expectedTurnOperation + || fence.kind === 'steer' && !isSafeIdentifier(fence.operationId)) stale(); +} + +function assertPauseAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + if ((state.status !== 'pause_requested' && state.status !== 'paused') + || fence.operationId !== controlOperationId('pause', state)) stale(); +} + +function assertCancelAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + if (state.status !== 'cancelling' || state.cancellationIntent?.cancellationId !== fence.operationId + || state.providerBarrierIntent?.phase !== 'published') stale(); +} + +function assertResumeAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + const intent = state.resumeIntent; + const completed = state.completedResume; + const callablePhase = intent?.phase === 'provider_in_doubt' + || (intent?.phase === 'settled' && completed?.operationId === intent.operationId + && completed.operationGeneration === intent.operationGeneration + && completed.kind === intent.kind && completed.controllerEpoch === intent.controllerEpoch); + if (!intent || intent.operationId !== fence.operationId || intent.operationGeneration !== fence.generation + || !callablePhase || intent.leaseExpiresAt !== fence.leaseExpiresAt + || expired(intent.leaseExpiresAt) + || (fence.executionId !== undefined && intent.executionId !== fence.executionId) + || (fence.attemptId !== undefined && intent.attemptId !== fence.attemptId)) stale(); +} + +function assertRecoveryAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + const attempt = state.recoveryAttempt; + if (!attempt || attempt.operationToken !== fence.operationId + || attempt.operationGeneration !== fence.generation || attempt.phase !== 'provider_in_doubt' + || attempt.leaseExpiresAt !== fence.leaseExpiresAt || expired(attempt.leaseExpiresAt) + || attempt.executionId !== fence.executionId || attempt.attemptId !== fence.attemptId) stale(); +} + +function assertModelAuthority(state: GoalSessionState, fence: GoalProviderOperationFence): void { + const intent = modelIntents(state).find(candidate => + compositeOperationId('model', candidate.modelChangeId, candidate.applicationToken ?? 'unclaimed') === fence.operationId); + if (!intent || !intent.applicationToken || intent.applicationControllerEpoch !== fence.controllerEpoch + || (intent.phase !== 'provider_in_doubt' && intent.phase !== 'committed') + || intent.leaseExpiresAt !== fence.leaseExpiresAt || expired(intent.leaseExpiresAt)) stale(); +} + +function modelIntents(state: GoalSessionState): GoalModelChangeIntent[] { + return state.modelChangeIntents ?? (state.modelChangeIntent ? [state.modelChangeIntent] : []); +} + +function terminal(state: GoalSessionState): boolean { + return state.status === 'terminated' || state.status === 'failed'; +} + +function expired(value: string | undefined): boolean { + return value !== undefined && Date.parse(value) <= Date.now(); +} + +function stale(): never { + throw new StaleGoalSessionFenceError('Provider first effect was durably invalidated'); +} diff --git a/packages/core/src/agents/goalSession/providerOpenFailure.ts b/packages/core/src/agents/goalSession/providerOpenFailure.ts new file mode 100644 index 000000000..4c3f5b4b7 --- /dev/null +++ b/packages/core/src/agents/goalSession/providerOpenFailure.ts @@ -0,0 +1,29 @@ +import type { GoalSessionState, GoalSessionStatePort } from './contract.js'; +import { GoalSessionContractError, StaleGoalSessionFenceError } from './errors.js'; +import { safeFailureDiagnostic } from './securityBoundary.js'; +import { nextState } from './support.js'; + +export async function throwPersistedProviderOpenFailure( + statePort: GoalSessionStatePort, + state: GoalSessionState, + error: unknown, +): Promise { + if (error instanceof GoalSessionContractError && error.code === 'PROVIDER_OPEN_IN_DOUBT') { + const failed = await statePort.compareAndSet(state, nextState(state, { + status: 'failed', initializationIntent: undefined, + failureReason: 'Codex provider open is durably in doubt; automatic reopen is disabled', + })); + if (!failed) throw new StaleGoalSessionFenceError( + 'Session ownership changed while provider-open doubt was being persisted', + ); + throw error; + } + if (error instanceof StaleGoalSessionFenceError || error instanceof GoalSessionContractError) throw error; + await statePort.compareAndSet(state, nextState(state, { + status: 'failed', + failureReason: safeFailureDiagnostic( + error instanceof Error ? error.message : '', 'Unable to create or resume provider session safely', + ), + })); + throw error; +} diff --git a/packages/core/src/agents/goalSession/providerOperationBoundary.ts b/packages/core/src/agents/goalSession/providerOperationBoundary.ts new file mode 100644 index 000000000..a0d83addb --- /dev/null +++ b/packages/core/src/agents/goalSession/providerOperationBoundary.ts @@ -0,0 +1,202 @@ +import type { + GoalExecutionIdentity, + GoalModelChangeAcknowledgement, + GoalRepositoryIdentity, + GoalSessionIdentity, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +export interface GoalProviderBarrierIntent { + generation: number; + operationId: string; + kind: 'cancellation' | 'terminal' | 'replacement' | 'lease_expiry'; + phase: 'pending' | 'published'; + claimedAt: string; + pendingCancellationId?: string; +} + +export interface GoalModelInvocationEvidence extends GoalExecutionIdentity { + modelChangeId: string; + generation: number; + occurrenceId: string; + requestedModel: string; + effectiveModel: string; + acceptedAt: string; +} + +export interface GoalUsageAccounting { + version: 1; + lastWatermark: number; + occurrences: string[]; +} + +export interface GoalProviderDuplexTransport { + readonly output: AsyncIterable; + write(message: string): Promise; + closeInput(): void; + cancel(): Promise; + readonly completion: Promise<{ exitCode: number | null }>; +} + +export interface GoalProviderOpenContext extends GoalExecutionIdentity { + repository: GoalRepositoryIdentity; + requestedModel: string; + providerHomeTarget: string; + credentialTargets: string[]; + /** Supervisor-minted durable key binding response-loss adoption. */ + deterministicOpenKey?: string; + transport: GoalProviderDuplexTransport; +} + +/** + * Serializable capability presented to the provider primitive at its first + * external effect. It contains no process-local callback or object identity. + * The provider atomically compares generation against its durable high-water + * mark and rejects a lower generation before opening a process, stream, socket, + * repository, container, or remote request. + * A present leaseExpiresAt is compared in that same atomic effect transaction. + */ +export interface GoalProviderOperationFence extends GoalSessionIdentity { + readonly controllerEpoch: number; + readonly generation: number; + readonly operationId: string; + readonly kind: 'open' | 'turn' | 'resume' | 'reconcile' | 'steer' | 'model' | 'pause' | 'cancel'; + readonly leaseExpiresAt?: string; + readonly turnId?: string; + readonly executionId?: string; + readonly attemptId?: string; +} + +/** Closed identity for one real external stage within a logical operation. */ +export type GoalProviderEffectStage = 'provider_primitive' | 'stream_first_next' | 'container_spawn'; + +const PROVIDER_EFFECT_STAGES: ReadonlySet = new Set([ + 'provider_primitive', 'stream_first_next', 'container_spawn', +]); +const PROVIDER_OPERATION_KINDS: ReadonlySet = new Set([ + 'open', 'turn', 'resume', 'reconcile', 'steer', 'model', 'pause', 'cancel', +]); +const PROVIDER_FENCE_FIELDS = new Set([ + 'goalId', 'sessionId', 'controllerEpoch', 'generation', 'operationId', 'kind', + 'leaseExpiresAt', 'turnId', 'executionId', 'attemptId', +]); + +/** Runtime boundary for JavaScript callers and values decoded from persistence. */ +export function assertGoalProviderEffectStage(value: unknown): asserts value is GoalProviderEffectStage { + if (typeof value !== 'string' || !PROVIDER_EFFECT_STAGES.has(value)) { + throw new GoalSessionContractError( + 'Provider effect stage is not one of the three internal stages', 'INVALID_PROVIDER_FENCE', + ); + } +} + +/** Closed runtime decoder for provider fences received from JS and persistence. */ +export function assertGoalProviderOperationFence(value: unknown): asserts value is GoalProviderOperationFence { + let descriptors: PropertyDescriptorMap; + try { + if (!value || typeof value !== 'object' || Array.isArray(value) + || ![Object.prototype, null].includes(Object.getPrototypeOf(value))) invalidFence(); + descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.getOwnPropertySymbols(value).length > 0 + || Object.entries(descriptors).some(([key, descriptor]) => + !PROVIDER_FENCE_FIELDS.has(key) || !descriptor.enumerable || !('value' in descriptor))) invalidFence(); + } catch (error) { + if (error instanceof GoalSessionContractError) throw error; + invalidFence(); + } + const field = (name: string): unknown => descriptors[name]?.value; + for (const name of ['goalId', 'sessionId', 'operationId']) { + if (!isSafeIdentifier(field(name))) invalidFence(); + } + const kind = field('kind'); + if (typeof kind !== 'string' || !PROVIDER_OPERATION_KINDS.has(kind)) invalidFence(); + for (const name of ['controllerEpoch', 'generation']) { + const number = field(name); + if (!Number.isSafeInteger(number) || (number as number) < 0) invalidFence(); + } + for (const name of ['turnId', 'executionId', 'attemptId']) { + const candidate = field(name); + if (candidate !== undefined && !isSafeIdentifier(candidate)) invalidFence(); + } + const leaseExpiresAt = field('leaseExpiresAt'); + if (leaseExpiresAt !== undefined + && (typeof leaseExpiresAt !== 'string' || !Number.isFinite(Date.parse(leaseExpiresAt)))) invalidFence(); + if ((kind === 'turn' || kind === 'steer') + && (!field('turnId') || !field('executionId') || !field('attemptId'))) invalidFence(); + if (kind === 'reconcile' && (!field('executionId') || !field('attemptId'))) invalidFence(); +} + +function invalidFence(): never { + throw new GoalSessionContractError('Provider operation fence is invalid', 'INVALID_PROVIDER_FENCE'); +} + +export interface GoalStartedProviderEffectCleanup { + readonly kind: 'rollback_or_cancel'; + readonly run: () => void | Promise; +} + +/** + * Proof that a provider primitive was synchronously and irrevocably started. + * The authoritative transaction is committed after this handle is returned; + * only its completion is awaited after the transaction has released its lock. + */ +export interface GoalStartedProviderEffect { + readonly completion: Promise; + /** Owns the already-started primitive if its authoritative transaction fails. */ + readonly cleanup: GoalStartedProviderEffectCleanup; +} + +/** + * Linearizes a provider primitive's first external effect with state invalidation. + * A production implementation must execute `effect` while holding the same + * database transaction/serialization lock used by GoalSessionStatePort writes. + * The callback must start the primitive synchronously and return an explicit + * non-Promise handle. Implementations reject callbacks that escape through an + * async return, commit after the handle is obtained, and await completion only + * after releasing the authoritative transaction. + */ +export interface GoalProviderFirstEffectPort { + start( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + effect: () => GoalStartedProviderEffect, + /** Rebuilds a fresh, bounded operation-specific DTO before settlement. */ + rebuild: (value: T) => R, + ): Promise; +} + +/** Monotonic provider-visible high-water publication. */ +export interface GoalProviderBarrierPublication extends GoalSessionIdentity { + readonly generation: number; + readonly publishedAt: string; + /** Cancellation remains addressable after a bounded caller timeout. */ + readonly pendingCancellationId?: string; +} + +export interface GoalModelChangeHistoryRecord { + operationId: string; + model: string; + /** Atomically allocated, unique and strictly increasing within goal/session scope. */ + sequence: number; + status: 'pending' | 'settled' | 'retired'; + acknowledgement?: GoalModelChangeAcknowledgement; +} + +/** + * Exact durable addressability ledger, stored separately from bounded session + * state. claim allocates sequence in the same transaction as the unique + * (scope, operationId) insert. Concurrent callers either observe the one exact + * row or retry a uniqueness/busy conflict; they never derive ordering from an + * aggregate read. settle atomically records the acknowledgement and retires + * settled rows older than the deterministic newest 64. Retired exact + * tombstones remain addressable and never use probabilistic membership. + */ +export interface GoalModelChangeHistoryPort { + claim(identity: GoalSessionIdentity, operationId: string, model: string): Promise; + settle( + identity: GoalSessionIdentity, + operationId: string, + acknowledgement: GoalModelChangeAcknowledgement, + ): Promise; +} diff --git a/packages/core/src/agents/goalSession/providerProtocolDuplex.ts b/packages/core/src/agents/goalSession/providerProtocolDuplex.ts new file mode 100644 index 000000000..256c9348c --- /dev/null +++ b/packages/core/src/agents/goalSession/providerProtocolDuplex.ts @@ -0,0 +1,124 @@ +import type { SupervisedDockerExecution, SupervisedDockerOutput } from '../../claude/docker/dockerExecutor.js'; +import type { GoalContainerOutputObserver } from './GoalContainerSupervisor.js'; +import type { GoalProviderDuplexTransport } from './providerOperationBoundary.js'; +import { GoalSessionContractError } from './errors.js'; + +const DEFAULT_PROTOCOL_QUEUE_BYTES = 2 * 1024 * 1024; + +/** + * One-use bridge from the isolated container's exact stdout chunks to a + * provider parser. Raw bytes live only in this bounded in-memory queue. They + * are never copied into durable events, JSONL, errors, or host-log tails. + */ +export function createProviderProtocolDuplex( + maxQueuedBytes = DEFAULT_PROTOCOL_QUEUE_BYTES, +): { + observer: GoalContainerOutputObserver; + bindExecution(execution: SupervisedDockerExecution): void; + transport: GoalProviderDuplexTransport; +} { + if (!Number.isSafeInteger(maxQueuedBytes) || maxQueuedBytes <= 0) { + throw new GoalSessionContractError('Protocol queue bound is invalid', 'UNSAFE_PROVIDER_VALUE'); + } + const queue: string[] = []; + const waiters: Array<{ + resolve(result: IteratorResult): void; + reject(error: GoalSessionContractError): void; + }> = []; + let queuedBytes = 0; + let execution: SupervisedDockerExecution | undefined; + let ended = false; + let failure: GoalSessionContractError | undefined; + let subscribed = true; + + const finish = (error?: GoalSessionContractError): void => { + if (ended) return; + ended = true; + failure = error; + while (waiters.length) { + const waiter = waiters.shift()!; + if (error) waiter.reject(error); + else waiter.resolve({ done: true, value: undefined }); + } + }; + const next = async (): Promise> => { + if (failure) throw failure; + const value = queue.shift(); + if (value !== undefined) { + queuedBytes -= Buffer.byteLength(value); + return { done: false, value }; + } + if (ended) return { done: true, value: undefined }; + return new Promise((resolve, reject) => waiters.push({ resolve, reject })); + }; + const push = (output: Readonly): void | 'unsubscribe' => { + if (!subscribed || ended) return 'unsubscribe'; + if (output.channel !== 'stdout') return; + const chunk = output.data; + const bytes = Buffer.byteLength(chunk); + if (queuedBytes + bytes > maxQueuedBytes) { + subscribed = false; + const error = new GoalSessionContractError( + 'Provider protocol exceeded its bounded in-memory queue', 'PROVIDER_OPERATION_FAILED', + ); + finish(error); + void execution?.cancel(error).catch(() => undefined); + return 'unsubscribe'; + } + const waiter = waiters.shift(); + if (waiter) waiter.resolve({ done: false, value: chunk }); + else { + queue.push(chunk); + queuedBytes += bytes; + } + }; + + const observer: GoalContainerOutputObserver = { + next: push, + complete: () => finish(), + error: () => finish(new GoalSessionContractError( + 'Supervised provider protocol failed safely', 'PROVIDER_OPERATION_FAILED', + )), + }; + const transport: GoalProviderDuplexTransport = { + output: { + [Symbol.asyncIterator]: () => ({ + next, + return: async () => { + subscribed = false; + finish(); + return { done: true, value: undefined }; + }, + }), + }, + async write(message: string): Promise { + if (!execution) throw new GoalSessionContractError( + 'Provider protocol transport is not bound', 'PROVIDER_OPERATION_FAILED', + ); + await execution.writeInput(message); + }, + closeInput(): void { execution?.closeInput(); }, + async cancel(): Promise { + subscribed = false; + finish(); + await execution?.cancel(new GoalSessionContractError( + 'Provider protocol cancelled safely', 'PROVIDER_OPERATION_FAILED', + )); + }, + get completion() { + return execution?.completion ?? Promise.reject(new GoalSessionContractError( + 'Provider protocol transport is not bound', 'PROVIDER_OPERATION_FAILED', + )); + }, + }; + return { + observer, + bindExecution(value): void { + if (execution) throw new GoalSessionContractError( + 'Provider protocol transport is already bound', 'PROVIDER_OPERATION_FAILED', + ); + execution = value; + }, + transport, + }; +} diff --git a/packages/core/src/agents/goalSession/providerResultBoundary.ts b/packages/core/src/agents/goalSession/providerResultBoundary.ts new file mode 100644 index 000000000..3be559ea8 --- /dev/null +++ b/packages/core/src/agents/goalSession/providerResultBoundary.ts @@ -0,0 +1,182 @@ +import type { + GoalModelChangeAcknowledgement, + GoalPauseAcknowledgement, + GoalProviderReconcileResult, + GoalProviderSessionSnapshot, + GoalSessionEvent, + GoalSessionJsonValue, +} from './contract.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; +import { + GoalSessionContractError, isProviderOpenInDoubtError, StaleGoalSessionFenceError, +} from './errors.js'; +import { sanitizeNewRecoveryMetadata } from './recoveryMetadata.js'; +import { safeProviderException, sanitizeGoalSessionEvent } from './securityBoundary.js'; + +type ClosedRecord = Record; + +export async function untrustedProviderResult( + effect: () => T | Promise, + rebuild: (value: Awaited) => R, +): Promise { + try { + return rebuild(await effect()); + } catch (error) { + if (error instanceof StaleGoalSessionFenceError) throw error; + if (isProviderOpenInDoubtError(error)) throw error; + throw safeProviderException(error); + } +} + +/** + * Provider values are capabilities, not data. This module evaluates every + * proxy trap/accessor and reconstructs a fresh, closed DTO while the call is + * still inside GoalSessionCore's untrusted-provider try/catch. + */ +export function rebuildProviderSnapshot(value: unknown, provider: string): GoalProviderSessionSnapshot { + const input = closedRecord(value, ['providerSessionId', 'recoveryMetadata', 'model'], 'session snapshot'); + const result: GoalProviderSessionSnapshot = { + providerSessionId: providerId(input.providerSessionId, 'providerSessionId'), + recoveryMetadata: sanitizeNewRecoveryMetadata(input.recoveryMetadata as GoalSessionJsonValue, provider), + }; + if (input.model !== undefined) result.model = providerId(input.model, 'model'); + return result; +} + +export function rebuildPauseAcknowledgement(value: unknown): GoalPauseAcknowledgement { + const input = closedRecord(value, ['appliesAt', 'boundaryReached'], 'pause acknowledgement'); + const result: GoalPauseAcknowledgement = { + appliesAt: closed(input.appliesAt, ['immediate', 'next_safe_boundary', 'after_turn'], 'pause boundary'), + }; + if (input.boundaryReached !== undefined) { + const boundary = closedRecord(input.boundaryReached, ['boundary', 'checkpointId'], 'pause boundary evidence'); + result.boundaryReached = { + boundary: providerId(boundary.boundary, 'pause boundary'), + checkpointId: boundary.checkpointId === undefined + ? undefined : providerId(boundary.checkpointId, 'pause checkpoint'), + }; + } + return result; +} + +export function rebuildMessageAcknowledgement(value: unknown): { messageId: string } { + const input = closedRecord(value, ['messageId'], 'message acknowledgement'); + return { messageId: providerId(input.messageId, 'messageId') }; +} + +/** Closed replay shape for provider primitives whose contract returns void. */ +export function rebuildVoidProviderResult(value: unknown): undefined { + if (value !== undefined && value !== null) malformed('void result'); + return undefined; +} + +export function rebuildModelAcknowledgement(value: unknown): GoalModelChangeAcknowledgement { + const input = closedRecord(value, ['outcome', 'requestedModel', 'appliesAt', 'effectiveModel'], 'model acknowledgement'); + const result: GoalModelChangeAcknowledgement = { + requestedModel: providerId(input.requestedModel, 'requestedModel'), + appliesAt: closed(input.appliesAt, ['immediate', 'next_safe_boundary', 'next_turn'], 'model boundary'), + }; + if (input.outcome !== undefined) { + result.outcome = closed( + input.outcome, ['acknowledged', 'outside_retry_horizon'] as const, 'model outcome', + ); + } + if (input.effectiveModel !== undefined) result.effectiveModel = providerId(input.effectiveModel, 'effectiveModel'); + return result; +} + +export function rebuildReconcileResult(value: unknown, provider: string): GoalProviderReconcileResult { + const input = closedRecord(value, ['outcome', 'snapshot', 'reason'], 'reconciliation result'); + const outcome = closed(input.outcome, ['alive', 'resumed', 'failed'], 'reconciliation outcome'); + // Provider prose is intentionally discarded. Public state receives only a + // closed code-derived sentence, never a provider-controlled diagnostic. + const reason = outcome === 'alive' + ? 'Provider reconciliation confirmed live work' + : outcome === 'resumed' + ? 'Provider reconciliation resumed durable work' + : 'Provider reconciliation failed safely'; + if (outcome === 'failed') { + if (input.snapshot !== undefined) malformed('failed reconciliation snapshot'); + return { outcome, reason }; + } + if (outcome === 'resumed' && input.snapshot === undefined) malformed('resumed reconciliation snapshot'); + const snapshot = input.snapshot === undefined ? undefined : rebuildProviderSnapshot(input.snapshot, provider); + return outcome === 'resumed' + ? { outcome, snapshot: snapshot!, reason } + : { outcome, snapshot, reason }; +} + +export interface RebuiltProviderIterator { + next(): Promise; + return?(): Promise; +} + +export function rebuildIterator(value: unknown): RebuiltProviderIterator { + const iterator = closedCapability(value, 'provider iterator'); + const next = method(iterator, 'next'); + const returnMethod = optionalMethod(iterator, 'return'); + return { + next: () => next.call(iterator), + return: returnMethod ? () => returnMethod.call(iterator) : undefined, + }; +} + +export function rebuildIteratorResult(value: unknown): IteratorResult { + const input = closedRecord(value, ['done', 'value'], 'iterator result'); + if (typeof input.done !== 'boolean') malformed('iterator done'); + if (input.done) return { done: true, value: undefined }; + if (input.value === undefined) malformed('iterator value'); + return { done: false, value: sanitizeGoalSessionEvent(input.value as GoalSessionEvent) }; +} + +function closedRecord(value: unknown, allowedFields: readonly string[], name: string): ClosedRecord { + if (!value || typeof value !== 'object' || Array.isArray(value)) malformed(name); + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) malformed(name); + const descriptors = Object.getOwnPropertyDescriptors(value); + const allowed = new Set(allowedFields); + if (Object.getOwnPropertySymbols(value).length > 0 + || Object.entries(descriptors).some(([key, descriptor]) => + !allowed.has(key) || !descriptor.enumerable || !('value' in descriptor))) malformed(name); + const result: ClosedRecord = {}; + for (const field of allowedFields) { + const descriptor = descriptors[field]; + if (descriptor && 'value' in descriptor) result[field] = descriptor.value; + } + return result; +} + +function closedCapability(value: unknown, name: string): ClosedRecord { + if (!value || typeof value !== 'object') malformed(name); + // Iterator methods normally live on a prototype. Reading descriptors walks + // that chain without invoking getters; a Proxy trap remains inside the + // provider boundary and is converted to the generic provider exception. + return value as ClosedRecord; +} + +function method(value: ClosedRecord, name: string): (...args: unknown[]) => Promise { + const candidate = Reflect.get(value, name); + if (typeof candidate !== 'function') malformed(`provider ${name}`); + return candidate as (...args: unknown[]) => Promise; +} + +function optionalMethod(value: ClosedRecord, name: string): ((...args: unknown[]) => Promise) | undefined { + const candidate = Reflect.get(value, name); + if (candidate === undefined) return undefined; + if (typeof candidate !== 'function') malformed(`provider ${name}`); + return candidate as (...args: unknown[]) => Promise; +} + +function providerId(value: unknown, name: string): string { + if (!isSafeIdentifier(value)) malformed(name); + return value; +} + +function closed(value: unknown, allowed: readonly T[], name: string): T { + if (typeof value !== 'string' || !allowed.includes(value as T)) malformed(name); + return value as T; +} + +function malformed(name: string): never { + throw new GoalSessionContractError(`Provider returned an invalid ${name}`, 'INVALID_PROVIDER_RESULT'); +} diff --git a/packages/core/src/agents/goalSession/reconcileRecoveredTurn.ts b/packages/core/src/agents/goalSession/reconcileRecoveredTurn.ts new file mode 100644 index 000000000..a971e3755 --- /dev/null +++ b/packages/core/src/agents/goalSession/reconcileRecoveredTurn.ts @@ -0,0 +1,34 @@ +import type { + GoalExecutionIdentity, + GoalProviderReconcileResult, + GoalSessionState, + GoalSessionStatus, + GoalTurnState, +} from './contract.js'; + +/** + * Applies a proven reconciliation outcome to the active turn. Merely observing + * the old container as alive leaves its identity untouched. Only a resumed + * outcome promotes the fresh reconciliation attempt to authoritative state. + */ +export function reconcileRecoveredTurn( + state: GoalSessionState, + execution: GoalExecutionIdentity, + outcome: GoalProviderReconcileResult['outcome'], +): { status: GoalSessionStatus; activeTurn: GoalTurnState | undefined } { + if (outcome === 'failed') return { status: 'failed', activeTurn: state.activeTurn }; + if (outcome !== 'resumed') return { status: state.status, activeTurn: state.activeTurn }; + const turn = state.activeTurn; + if (turn && (turn.status === 'running' || turn.status === 'pause_requested' || turn.status === 'paused')) { + return { + status: 'paused', + activeTurn: { + ...turn, + ...execution, + executionEpoch: state.controllerEpoch, + status: 'paused', + }, + }; + } + return { status: 'idle', activeTurn: turn }; +} diff --git a/packages/core/src/agents/goalSession/reconciliationIdentity.ts b/packages/core/src/agents/goalSession/reconciliationIdentity.ts new file mode 100644 index 000000000..35c4a7206 --- /dev/null +++ b/packages/core/src/agents/goalSession/reconciliationIdentity.ts @@ -0,0 +1,119 @@ +import type { + GoalContainerInspection, + GoalRepositoryIdentity, + GoalRepositoryInspection, + GoalSessionState, +} from './contract.js'; +import { fingerprintGoalWorktree, normalizeGitRepositoryIdentity } from './worktreeIdentity.js'; +import path from 'node:path'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +const SHA = /^[a-f\d]{4,64}$/i; +const FINGERPRINT = /^[a-f\d]{64}$/i; +const SAFE_BRANCH = /^(?![./])(?!.*(?:\.\.|@\{|\\|\s|[~^:?*]|\[))(?!.*\.$)[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$/; + +/** Removes untrusted recovery-port fields before provider or audit boundaries. */ +export function sanitizeRepositoryInspection( + expected: GoalRepositoryIdentity, + inspection: GoalRepositoryInspection, +): GoalRepositoryInspection { + const observedRepository = inspection.observedRepository === undefined + ? undefined + : normalizeGitRepositoryIdentity(inspection.observedRepository); + const invalidRemote = inspection.observedRepository !== undefined && !observedRepository; + return { + ...expected, + exists: inspection.exists === true, + dirty: inspection.dirty === true, + observedRepository, + observedHeadSha: SHA.test(inspection.observedHeadSha ?? '') ? inspection.observedHeadSha : undefined, + observedBranch: !invalidRemote && SAFE_BRANCH.test(inspection.observedBranch ?? '') + ? inspection.observedBranch : undefined, + observedWorktreeFingerprint: invalidRemote + ? undefined + : FINGERPRINT.test(inspection.observedWorktreeFingerprint ?? '') + ? inspection.observedWorktreeFingerprint + : undefined, + resolvedWorktreePath: inspection.resolvedWorktreePath === path.resolve(expected.worktreePath) + ? inspection.resolvedWorktreePath : undefined, + reason: invalidRemote ? 'Git remote does not contain a trustworthy repository identity' + : inspection.reason ? 'Repository inspection did not establish an authoritative checkout' : undefined, + }; +} + +/** Explicit allowlist for the untrusted recovery-port container result. */ +export function sanitizeContainerInspection(inspection: GoalContainerInspection): GoalContainerInspection { + const status = ['running', 'exited', 'missing', 'daemon_unavailable'].includes(inspection.status) + ? inspection.status : 'daemon_unavailable'; + const identity = inspection.recoveryIdentity; + const recoveryIdentity = identity + && isSafeIdentifier(identity.goalId) && isSafeIdentifier(identity.sessionId) + && isSafeIdentifier(identity.turnId) && isSafeIdentifier(identity.attemptId) + && Number.isSafeInteger(identity.executionEpoch) && identity.executionEpoch >= 0 + && FINGERPRINT.test(identity.worktreeFingerprint) + ? { + goalId: identity.goalId, sessionId: identity.sessionId, + executionEpoch: identity.executionEpoch, turnId: identity.turnId, + attemptId: identity.attemptId, worktreeFingerprint: identity.worktreeFingerprint, + } : undefined; + return { + status, + containerId: isSafeIdentifier(inspection.containerId) ? inspection.containerId : undefined, + containerName: isSafeIdentifier(inspection.containerName) ? inspection.containerName : undefined, + recoveryIdentity, + reason: inspection.reason ? 'Container inspection did not establish an authoritative runtime' : undefined, + }; +} + +export function verifyRecoveredContainer( + state: GoalSessionState, + inspection: GoalContainerInspection, + worktreeFingerprint: string, +): string | null { + if (inspection.status === 'missing') return null; + if (inspection.status === 'daemon_unavailable') { + return `Container identity could not be inspected: ${inspection.reason ?? 'Docker unavailable'}`; + } + const turn = state.activeTurn; + if (!turn) return 'A recovered container exists without an authoritative active turn'; + const observed = inspection.recoveryIdentity; + if (!observed) return 'Recovered container is missing authoritative recovery metadata'; + const expected = { + goalId: state.goalId, + sessionId: state.sessionId, + executionEpoch: turn.executionEpoch, + turnId: turn.turnId, + attemptId: turn.attemptId, + worktreeFingerprint, + }; + for (const key of Object.keys(expected) as Array) { + if (observed[key] !== expected[key]) { + return `Recovered container ${key} does not match authoritative identity`; + } + } + return null; +} + +/** Verifies authoritative checkout identity while allowing legitimate HEAD advancement. */ +export function verifyReconciliationTarget( + expected: GoalRepositoryIdentity, + inspection: GoalRepositoryInspection, +): string | null { + if (!inspection.exists) { + return 'Authoritative goal worktree is unavailable'; + } + if (!inspection.observedBranch) { + return 'Authoritative worktree branch could not be observed'; + } + const expectedFingerprint = fingerprintGoalWorktree(expected); + if (!inspection.observedWorktreeFingerprint) { + return 'Authoritative worktree fingerprint could not be observed'; + } + if (inspection.observedWorktreeFingerprint !== expectedFingerprint) { + return 'Worktree fingerprint mismatch against authoritative identity'; + } + if (inspection.observedBranch !== expected.branch) { + return 'Worktree branch mismatch against authoritative identity'; + } + return null; +} diff --git a/packages/core/src/agents/goalSession/recoveryMetadata.ts b/packages/core/src/agents/goalSession/recoveryMetadata.ts new file mode 100644 index 000000000..e45cfa215 --- /dev/null +++ b/packages/core/src/agents/goalSession/recoveryMetadata.ts @@ -0,0 +1,252 @@ +import type { GoalSessionJsonValue } from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +export const GOAL_RECOVERY_METADATA_CODEC_VERSION = 2; +const MAX_ENVELOPE_BYTES = 32 * 1024; +const MAX_USAGE_COMPONENTS = 32; +const SECRET_VALUE = /(?:Bearer\s*\S+|gh[oprsu]_|github_pat_|sk-|AKIA|secret|token|password|credential|private.?key|https?:\/\/[^\s]*@|ssh:\/\/[^\s]*@|-----BEGIN)/i; +const SENSITIVE_FIELD = /(?:secret|token|password|credential|authorization|private.?key|api.?key)/i; + +type RecoveryProvider = 'codex' | 'claude' | 'antigravity'; +/** Legacy durable shape retained for source compatibility during v1 migration. */ +export interface GoalRecoveryMetadataV1 { + version?: 1; + checkpoint?: string; + conversation?: string; + cursor?: string | number; + offset?: number; + sequence?: number; + revision?: string | number; + phase?: string; + state?: string; +} +type Codec = { protocolVersion: string; required: readonly string[]; optional: readonly string[] }; + +/** Pinned provider protocol codecs accepted by the v2 envelope. */ +const PROVIDER_CODECS: Readonly> = { + codex: { + protocolVersion: 'app-server-0.146.0', + required: ['threadId', 'initialized'], + optional: [ + 'sessionId', 'turnId', 'checkpoint', 'openKey', 'repository', 'model', + 'providerHomeIdentity', 'cliVersion', + ], + }, + claude: { + protocolVersion: 'cli-2.1.220', + required: ['sessionId'], + optional: ['reasoningId', 'checkpoint', 'transcriptCursor'], + }, + antigravity: { + protocolVersion: 'cli-1.1.13', + required: ['conversationId', 'manifestVersion', 'manifestChecksum'], + optional: ['checkpoint', 'conversationChecksum'], + }, +}; + +const LEGACY_FIELDS = new Set([ + 'checkpoint', 'conversation', 'cursor', 'offset', 'sequence', 'revision', 'phase', 'state', 'version', +]); +const CLOSED_VALUES: Readonly>> = { + phase: new Set(['initialized', 'pending', 'running', 'paused', 'checkpoint', 'completed', 'failed', 'cancelled']), + state: new Set(['pending', 'active', 'idle', 'paused', 'completed', 'failed', 'cancelled', 'terminated']), +}; + +/** + * Decodes either the bounded provider-specific v2 envelope or an existing flat + * v1 record during migration. New provider state should always use v2; v1 is + * retained only so an already-durable session can be reopened deterministically. + */ +export function sanitizeRecoveryMetadata( + value: GoalSessionJsonValue, + expectedProvider?: string, +): GoalSessionJsonValue { + if (!isPlainObject(value)) invalid('Recovery metadata must be an object'); + assertBounded(value); + if (value.version === GOAL_RECOVERY_METADATA_CODEC_VERSION) { + return decodeV2(value, expectedProvider); + } + return decodeLegacyV1(value); +} + +/** + * New provider ingress is v2-only for the three pinned providers. Unknown + * embedding adapters retain their closed legacy codec for source compatibility; + * an existing v1 durable record may be read, but can never be returned by a + * successful Codex/Claude/Antigravity interaction. + */ +export function sanitizeNewRecoveryMetadata( + value: GoalSessionJsonValue, + expectedProvider: string, +): GoalSessionJsonValue { + const decoded = sanitizeRecoveryMetadata(value, expectedProvider); + if (expectedProvider === 'codex' || expectedProvider === 'claude' || expectedProvider === 'antigravity') { + if (!isPlainObject(decoded) || decoded.version !== GOAL_RECOVERY_METADATA_CODEC_VERSION + || !isPlainObject(decoded.payload)) invalid('New provider recovery metadata must use the pinned v2 codec'); + const payload = decoded.payload as Record; + const required = expectedProvider === 'codex' + ? [ + 'threadId', 'sessionId', 'initialized', 'openKey', 'repository', 'model', + 'providerHomeIdentity', 'cliVersion', + ] + : PROVIDER_CODECS[expectedProvider].required; + if (required.some(field => payload[field] === undefined)) { + invalid('New provider recovery metadata is missing an exact identity'); + } + } + return decoded; +} + +function decodeV2(value: Record, expectedProvider?: string): GoalSessionJsonValue { + exactFields(value, ['version', 'provider', 'protocolVersion', 'payload', 'usage']); + const provider = providerName(value.provider); + if (expectedProvider !== undefined && expectedProvider !== provider) { + invalid('Recovery metadata belongs to a different provider'); + } + const codec = PROVIDER_CODECS[provider]; + if (value.protocolVersion !== codec.protocolVersion) invalid('Recovery protocol version is unsupported'); + if (!isPlainObject(value.payload)) invalid('Recovery payload must be an object'); + exactFields(value.payload, [...codec.required, ...codec.optional]); + for (const field of codec.required) if (value.payload[field] === undefined) invalid(`Recovery payload is missing ${field}`); + const payload: Record = {}; + for (const field of [...codec.required, ...codec.optional]) { + const candidate = value.payload[field]; + if (candidate === undefined) continue; + payload[field] = field === 'initialized' + ? safeBoolean(candidate, field) + : field === 'manifestVersion' || field === 'transcriptCursor' + ? safeNonNegativeInteger(candidate, field) + : field === 'repository' + ? safeRepositoryIdentity(candidate) + : field === 'providerHomeIdentity' + ? safeProviderHomeIdentity(candidate) + : safeIdentifier(candidate, field); + } + return { + version: GOAL_RECOVERY_METADATA_CODEC_VERSION, + provider, + protocolVersion: codec.protocolVersion, + payload, + usage: decodeUsage(value.usage), + }; +} + +function decodeUsage(value: GoalSessionJsonValue | undefined): GoalSessionJsonValue { + if (value === undefined) return { components: [] }; + if (!isPlainObject(value)) invalid('Recovery usage must be an object'); + exactFields(value, ['components']); + if (!Array.isArray(value.components) || value.components.length > MAX_USAGE_COMPONENTS) { + invalid('Recovery usage components are invalid'); + } + const seen = new Set(); + const components = value.components.map((candidate, index) => { + if (!isPlainObject(candidate)) invalid(`Recovery usage component ${index} is invalid`); + exactFields(candidate, ['component', 'watermark', 'occurrenceId']); + const component = closed(candidate.component, ['input_tokens', 'output_tokens', 'cached_input_tokens', 'cost_usd'], 'usage component'); + if (seen.has(component)) invalid('Recovery usage component is duplicated'); + seen.add(component); + return { + component, + watermark: safeNonNegativeInteger(candidate.watermark, 'usage watermark'), + occurrenceId: safeIdentifier(candidate.occurrenceId, 'usage occurrenceId'), + }; + }); + return { components }; +} + +function decodeLegacyV1(value: Record): GoalSessionJsonValue { + const result: Record = {}; + for (const [key, candidate] of Object.entries(value)) { + if (!LEGACY_FIELDS.has(key)) rejectExtra(key, candidate); + if (key === 'version') { + if (candidate !== 1) invalid('Recovery metadata codec version is unsupported'); + result.version = 1; + continue; + } + if (key === 'offset' || key === 'sequence') result[key] = safeNonNegativeInteger(candidate, key); + else if ((key === 'cursor' || key === 'revision') && typeof candidate === 'number') result[key] = safeNonNegativeInteger(candidate, key); + else { + const decoded = safeIdentifier(candidate, key); + if (CLOSED_VALUES[key] && !CLOSED_VALUES[key].has(decoded)) invalid(`Recovery metadata contains an invalid ${key}`); + result[key] = decoded; + } + } + return result; +} + +function exactFields(value: Record, allowedFields: readonly string[]): void { + const allowed = new Set(allowedFields); + for (const [key, candidate] of Object.entries(value)) if (!allowed.has(key)) rejectExtra(key, candidate); +} + +function providerName(value: GoalSessionJsonValue): RecoveryProvider { + if (value !== 'codex' && value !== 'claude' && value !== 'antigravity') invalid('Recovery provider is unsupported'); + return value; +} + +function safeIdentifier(value: GoalSessionJsonValue | undefined, field: string): string { + if (!isSafeIdentifier(value) || SECRET_VALUE.test(value)) invalid(`Recovery metadata contains an invalid ${field}`); + return value; +} + +function safeNonNegativeInteger(value: GoalSessionJsonValue | undefined, field: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) invalid(`Recovery metadata contains an invalid ${field}`); + return value; +} + +function safeBoolean(value: GoalSessionJsonValue, field: string): boolean { + if (typeof value !== 'boolean') invalid(`Recovery metadata contains an invalid ${field}`); + return value; +} + +function safeRepositoryIdentity(value: GoalSessionJsonValue | undefined): string { + if (typeof value !== 'string' || !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(value) + || Buffer.byteLength(value) > 512 || SECRET_VALUE.test(value)) invalid('Recovery repository identity is invalid'); + return value; +} + +function safeProviderHomeIdentity(value: GoalSessionJsonValue | undefined): string { + if (value !== '/home/node/.codex' && value !== '/home/node/.claude' && value !== '/home/node/.gemini') { + invalid('Recovery provider-home identity is invalid'); + } + return value; +} + +function closed(value: GoalSessionJsonValue | undefined, allowed: readonly T[], field: string): T { + if (typeof value !== 'string' || !allowed.includes(value as T)) invalid(`Recovery metadata contains an invalid ${field}`); + return value as T; +} + +function isPlainObject(value: unknown): value is Record { + if (!value || Array.isArray(value) || typeof value !== 'object') return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function assertBounded(value: GoalSessionJsonValue): void { + let serialized: string; + try { serialized = JSON.stringify(value); } + catch { invalid('Recovery metadata is not serializable'); } + if (Buffer.byteLength(serialized) > MAX_ENVELOPE_BYTES) invalid('Recovery metadata exceeds its size bound'); +} + +function rejectExtra(key: string, value: GoalSessionJsonValue): never { + if (SENSITIVE_FIELD.test(key) || (typeof value === 'string' && SECRET_VALUE.test(value))) { + throw new GoalSessionContractError('Recovery metadata contains credential material', 'RECOVERY_METADATA_CONTAINS_CREDENTIAL'); + } + invalid('Recovery metadata contains an undeclared field'); +} + +function invalid(message: string): never { + throw new GoalSessionContractError(message, 'INVALID_RECOVERY_METADATA'); +} + +export function assertCredentialFreeRecoveryMetadata(value: GoalSessionJsonValue, expectedProvider?: string): void { + sanitizeRecoveryMetadata(value, expectedProvider); +} + +/** Existing callers use this name during reopen; strict decoding never scrubs. */ +export function scrubDurableRecoveryMetadata(value: GoalSessionJsonValue, expectedProvider?: string): GoalSessionJsonValue { + return sanitizeRecoveryMetadata(value, expectedProvider); +} diff --git a/packages/core/src/agents/goalSession/recoveryOperationProtocol.ts b/packages/core/src/agents/goalSession/recoveryOperationProtocol.ts new file mode 100644 index 000000000..4c7f96d93 --- /dev/null +++ b/packages/core/src/agents/goalSession/recoveryOperationProtocol.ts @@ -0,0 +1,79 @@ +import type { + GoalExecutionIdentity, GoalSessionState, +} from './contract.js'; +import { StaleGoalSessionFenceError } from './errors.js'; + +export const RECOVERY_LEASE_MS = 30_000; + +const RECOVERABLE_STATUSES = new Set([ + 'initializing', 'idle', 'running', 'pause_requested', 'paused', +]); + +export function isRecoverableStatus(status: GoalSessionState['status']): boolean { + return RECOVERABLE_STATUSES.has(status); +} + +export function stoppedReconciliationResult(state: GoalSessionState): { + outcome: 'blocked'; reason: string; state: GoalSessionState; +} | null { + if (isRecoverableStatus(state.status)) return null; + return { + outcome: 'blocked', + reason: state.status === 'cancelling' + ? 'Cancellation recovery must complete without reconciling provider work' + : `A ${state.status} session cannot be reconciled`, + state, + }; +} + +export function sameRecoverySubject(expected: GoalSessionState, current: GoalSessionState): boolean { + return expected.controllerEpoch === current.controllerEpoch + && isRecoverableStatus(current.status) + && expected.status === current.status + && expected.providerSessionId === current.providerSessionId + && expected.activeTurn?.turnId === current.activeTurn?.turnId + && expected.activeTurn?.executionId === current.activeTurn?.executionId + && expected.activeTurn?.attemptId === current.activeTurn?.attemptId + && expected.activeTurn?.status === current.activeTurn?.status + && expected.activeTurn?.executionEpoch === current.activeTurn?.executionEpoch; +} + +export function assertRecoverableExactState(state: GoalSessionState, controllerEpoch: number): void { + if (state.controllerEpoch !== controllerEpoch || !isRecoverableStatus(state.status)) { + throw new StaleGoalSessionFenceError('Session is no longer in an exact recoverable live state'); + } + const recovery = state.recoveryAttempt; + const changed = recovery?.authoritativeAttemptId !== undefined + && recovery.authoritativeAttemptId !== state.activeTurn?.attemptId + || recovery?.authoritativeExecutionId !== undefined + && recovery.authoritativeExecutionId !== state.activeTurn?.executionId + || recovery?.sessionStatus !== undefined && recovery.sessionStatus !== state.status + || recovery?.authoritativeTurnStatus !== undefined + && recovery.authoritativeTurnStatus !== state.activeTurn?.status; + if (changed) throw new StaleGoalSessionFenceError('The authoritative recovery subject changed'); +} + +export function assertLiveRecoveryLease( + state: GoalSessionState, + execution: GoalExecutionIdentity, + operationToken: string, +): void { + assertRecoverableExactState(state, state.controllerEpoch); + const recovery = state.recoveryAttempt; + if (!recovery || recovery.operationToken !== operationToken + || recovery.executionId !== execution.executionId || recovery.attemptId !== execution.attemptId + || recovery.phase !== 'provider_in_doubt' + || recovery.operationGeneration !== state.providerOperationGeneration + || Date.parse(recovery.leaseExpiresAt) <= Date.now()) { + throw new StaleGoalSessionFenceError('Reconciliation provider operation was durably preempted'); + } +} + +export function completedRecoveryResult(state: GoalSessionState, controllerEpoch: number): { + outcome: 'alive' | 'resumed' | 'failed'; reason: string; state: GoalSessionState; +} | null { + const recovery = state.completedRecovery; + if (state.controllerEpoch !== controllerEpoch || recovery?.controllerEpoch !== controllerEpoch + || state.status === 'cancelling' || state.status === 'terminated') return null; + return { outcome: recovery.outcome, reason: recovery.reason, state }; +} diff --git a/packages/core/src/agents/goalSession/recoveryRevalidation.ts b/packages/core/src/agents/goalSession/recoveryRevalidation.ts new file mode 100644 index 000000000..4c94bac0f --- /dev/null +++ b/packages/core/src/agents/goalSession/recoveryRevalidation.ts @@ -0,0 +1,30 @@ +import type { GoalSessionControlFence, GoalSessionState } from './contract.js'; +import { StaleGoalSessionFenceError } from './errors.js'; +import { sameRecoverySubject, stoppedReconciliationResult } from './recoveryOperationProtocol.js'; + +export class RecoveryGuardResult extends Error { + constructor(readonly result: T) { super('Recovery stopped while revalidating provider inspection'); } +} + +export async function revalidateRecoveryInspection(options: { + expected: GoalSessionState; + fence: GoalSessionControlFence; + load: () => Promise; + guard: (state: GoalSessionState) => Promise; +}): Promise { + const current = await options.load(); + const guarded = await options.guard(current); + if (guarded) throw new RecoveryGuardResult(guarded); + const revalidated = await options.load(); + const stopped = stoppedReconciliationResult(revalidated); + if (stopped) { + if (revalidated.status === 'cancelling') { + throw new RecoveryGuardResult((await options.guard(revalidated))!); + } + throw new RecoveryGuardResult(stopped as T); + } + if (!sameRecoverySubject(options.expected, revalidated)) { + throw new StaleGoalSessionFenceError('Recovery subject changed during durable inspection'); + } + return revalidated; +} diff --git a/packages/core/src/agents/goalSession/repositorySecurity.ts b/packages/core/src/agents/goalSession/repositorySecurity.ts new file mode 100644 index 000000000..e3e5e7fda --- /dev/null +++ b/packages/core/src/agents/goalSession/repositorySecurity.ts @@ -0,0 +1,35 @@ +import type { + GoalRepositoryIdentity, + GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { normalizeCanonicalGoalRepositoryIdentity } from './worktreeIdentity.js'; +import { assertSafeCallerTurnIdentity } from './safeIdentifier.js'; + +export function validateTurnRequestIdentity( + request: { turnId: string; executionId: string; attemptId?: string }, +): void { + assertSafeCallerTurnIdentity(request); +} + +export async function credentialFreeRepositoryIdentity(repositoryInput: GoalRepositoryIdentity): Promise { + const repository = await normalizeCanonicalGoalRepositoryIdentity(repositoryInput); + if (!repository) { + throw new GoalSessionContractError( + 'Repository identity is not a trustworthy Git repository name or remote', + 'INVALID_REPOSITORY', + ); + } + return repository; +} + +export async function normalizeRecoveryRepositories( + state: GoalSessionState, + requested: GoalRepositoryIdentity, +): Promise<{ requested: GoalRepositoryIdentity; durable: GoalRepositoryIdentity } | undefined> { + const normalizedRequested = await normalizeCanonicalGoalRepositoryIdentity(requested); + const normalizedDurable = await normalizeCanonicalGoalRepositoryIdentity(state.activeTurn?.repository ?? requested); + return normalizedRequested && normalizedDurable + ? { requested: normalizedRequested, durable: normalizedDurable } + : undefined; +} diff --git a/packages/core/src/agents/goalSession/runtimePorts.ts b/packages/core/src/agents/goalSession/runtimePorts.ts new file mode 100644 index 000000000..41d749dcb --- /dev/null +++ b/packages/core/src/agents/goalSession/runtimePorts.ts @@ -0,0 +1,28 @@ +import type { + GoalContainerInspection, GoalModelChangeHistoryPort, GoalProviderFirstEffectPort, GoalRepositoryIdentity, + GoalRepositoryInspection, GoalSessionEventSink, GoalSessionIdentity, + GoalSessionMessagePort, GoalSessionStatePort, GoalSessionTerminalPort, + GoalSessionTransitionPort, +} from './contract.js'; + +export interface GoalSessionRecoveryPort { + inspectContainer(identity: GoalSessionIdentity): Promise; + inspectRepository(repository: GoalRepositoryIdentity): Promise; +} + +export interface GoalSessionRuntimePorts { + state: GoalSessionStatePort; + transitions: GoalSessionTransitionPort; + events: GoalSessionEventSink; + terminal: GoalSessionTerminalPort; + messages: GoalSessionMessagePort; + recovery: GoalSessionRecoveryPort; + modelChanges: GoalModelChangeHistoryPort; + /** Same authoritative transaction domain as state; never a process-local mutex. */ + providerFirstEffects: GoalProviderFirstEffectPort; +} + +export type { + GoalProviderEffectClaimResult, GoalProviderEffectTransactionDomain, + GoalSessionAuthoritativeTransactionDomain, +} from './AuthoritativeGoalSessionRuntimePorts.js'; diff --git a/packages/core/src/agents/goalSession/safeIdentifier.ts b/packages/core/src/agents/goalSession/safeIdentifier.ts new file mode 100644 index 000000000..e02948927 --- /dev/null +++ b/packages/core/src/agents/goalSession/safeIdentifier.ts @@ -0,0 +1,37 @@ +import { GoalSessionContractError } from './errors.js'; + +/** Canonical grammar for #2018's 255-byte opaque runtime identifiers. */ +export const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,254}$/; + +/** Credential formats that remain syntactically valid opaque identifiers. */ +export const SECRET_ID_PREFIX = /^(?:Bearer|gh[oprsu]_|github_pat_|sk-|AKIA)/i; + +export function isSafeIdentifier(value: unknown): value is string { + return typeof value === 'string' && Buffer.byteLength(value, 'utf8') <= 255 + && SAFE_ID.test(value) && !SECRET_ID_PREFIX.test(value); +} + +export function assertSafeCallerTurnIdentity(request: { + turnId: unknown; + executionId: unknown; + attemptId?: unknown; +}): void { + if (!isSafeIdentifier(request.turnId) || !isSafeIdentifier(request.executionId) + || request.attemptId !== undefined && !isSafeIdentifier(request.attemptId)) { + throw new GoalSessionContractError( + 'turnId, executionId, and attemptId must be safe opaque identifiers', 'INVALID_TURN', + ); + } +} + +export function assertSafeCallerSteeringIdentity(request: { + turnId: unknown; + executionId?: unknown; + attemptId?: unknown; +}): void { + if (!isSafeIdentifier(request.turnId) + || request.executionId !== undefined && !isSafeIdentifier(request.executionId) + || request.attemptId !== undefined && !isSafeIdentifier(request.attemptId)) { + throw new GoalSessionContractError('Steering identity is unsafe', 'INVALID_TURN'); + } +} diff --git a/packages/core/src/agents/goalSession/securityBoundary.ts b/packages/core/src/agents/goalSession/securityBoundary.ts new file mode 100644 index 000000000..38a1af13f --- /dev/null +++ b/packages/core/src/agents/goalSession/securityBoundary.ts @@ -0,0 +1,166 @@ +import type { GoalSessionEvent } from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +import type { GoalSessionJsonValue } from './contract.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +const SECRET = /(?:Bearer\s*\S+|gh[oprsu]_|github_pat_|sk-|AKIA|secret|token|password|credential|private.?key|-----BEGIN|https?:\/\/[^\s]*@)/i; +const WINDOWS_OR_UNC = /^(?:[A-Za-z]:[\\/]|\\\\|\/\/)/; +const URI_OR_ENDPOINT = /^(?:file|https?|ssh|git|docker|podman|unix|tcp):/i; +const COMMAND_LIKE = /(?:^|\s)(?:sh|bash|zsh|cmd(?:\.exe)?|powershell|docker|podman|sudo)(?:\s|$)|[;&|`$<>]/i; + +export function safeDiagnostic(value: string, fallback: string): string { + const normalized = value.trim(); + return normalized && normalized.length <= 2048 && !SECRET.test(normalized) + && !/[\0\r]/.test(normalized) ? normalized : fallback; +} + +export function safeFailureDiagnostic(value: string, fallback: string): string { + const normalized = safeDiagnostic(value, fallback); + return /(?:^|\s)(?:\/|\.\.?[\\/])|[A-Za-z]:[\\/]|(?:file|https?|ssh|git|docker|podman|tcp|unix):\/\/|\S+@\S+:|\b(?:argv|command|mount|remote|endpoint|environment|config|docker|podman|npm|npx|yarn|pnpm)\b/i.test(normalized) + ? fallback : normalized.slice(0, 512); +} + +/** Rebuilds an untrusted provider exception without its stack, cause, or excess fields. */ +export function safeProviderException(error: unknown, fallback = 'Provider operation failed safely'): GoalSessionContractError { + // Never inspect an untrusted Error. message/cause/stack/name may be hostile + // getters and even a provider-created GoalSessionContractError is not a + // trusted internal contract error once it crosses this boundary. + void error; + return new GoalSessionContractError(fallback, 'PROVIDER_OPERATION_FAILED'); +} + +/** Copies only documented event fields; provider excess properties never cross persistence. */ +export function sanitizeGoalSessionEvent(event: GoalSessionEvent): GoalSessionEvent { + switch (event.type) { + case 'output': return { type: 'output', channel: closed(event.channel, ['stdout', 'stderr'], 'output channel'), data: safeOutput(event.data) }; + case 'assistant': return clean({ type: 'assistant', messageId: safeOptionalId(event.messageId), content: safeDiagnostic(event.content, '[redacted]'), data: safeJson(event.data) }); + case 'tool': return clean({ type: 'tool', toolCallId: safeId(event.toolCallId), name: safeId(event.name), phase: closed(event.phase, ['started', 'progress', 'completed', 'failed'], 'tool phase'), data: safeJson(event.data) }); + case 'todo': return clean({ type: 'todo', todoId: safeId(event.todoId), title: safeDiagnostic(event.title, '[redacted]'), status: closed(event.status, ['pending', 'in_progress', 'completed', 'cancelled'], 'todo status'), data: safeJson(event.data) }); + case 'usage': return clean({ type: 'usage', occurrenceId: safeId(event.occurrenceId), semantics: closed(event.semantics, ['delta', 'cumulative'], 'usage semantics'), watermark: requiredNonNegativeInteger(event.watermark, 'watermark'), model: safeOptionalId(event.model), inputTokens: nonNegativeInteger(event.inputTokens, 'inputTokens'), outputTokens: nonNegativeInteger(event.outputTokens, 'outputTokens'), cachedInputTokens: nonNegativeInteger(event.cachedInputTokens, 'cachedInputTokens'), costUsd: nonNegativeFinite(event.costUsd, 'costUsd'), data: safeJson(event.data) }); + case 'checkpoint': return clean({ type: 'checkpoint', checkpointId: safeId(event.checkpointId), recoveryMetadata: sanitizeRecoveryMetadata(event.recoveryMetadata), providerSessionId: safeOptionalId(event.providerSessionId) }); + case 'message_acknowledged': return { type: 'message_acknowledged', messageId: safeId(event.messageId) }; + case 'pause_requested': return { type: 'pause_requested', appliesAt: closed(event.appliesAt, ['immediate', 'next_safe_boundary', 'after_turn'], 'pause boundary') }; + case 'pause_boundary': return clean({ type: 'pause_boundary', boundary: safeId(event.boundary), checkpointId: safeOptionalId(event.checkpointId), providerEventId: safeOptionalId(event.providerEventId), providerEventOrdinal: nonNegativeInteger(event.providerEventOrdinal, 'providerEventOrdinal') }); + case 'session_resumed': return { type: 'session_resumed' }; + case 'model_change_acknowledged': return { type: 'model_change_acknowledged', requestedModel: safeId(event.requestedModel), appliesAt: closed(event.appliesAt, ['immediate', 'next_safe_boundary', 'next_turn'], 'model boundary') }; + case 'model_changed': return clean({ type: 'model_changed', previousModel: safeOptionalId(event.previousModel), model: safeId(event.model), providerEventId: safeOptionalId(event.providerEventId), providerEventOrdinal: nonNegativeInteger(event.providerEventOrdinal, 'providerEventOrdinal') }); + case 'turn_resumed': return { type: 'turn_resumed', turnId: safeId(event.turnId) }; + case 'reconciliation': return { type: 'reconciliation', outcome: closed(event.outcome, ['alive', 'resumed', 'failed', 'blocked'], 'reconciliation outcome'), reason: 'Provider reconciliation completed safely' }; + case 'completion': return clean({ type: 'completion', outcome: closed(event.outcome, ['succeeded', 'failed', 'cancelled'], 'completion outcome'), summary: event.summary ? '[redacted]' : undefined, error: event.error ? 'Provider operation failed safely' : undefined }); + } + throw new GoalSessionContractError('Provider emitted an unknown event type', 'INVALID_PROVIDER_EVENT'); +} + +function clean(value: T): T { + return Object.fromEntries(Object.entries(value).filter(([, nested]) => nested !== undefined)) as T; +} + +function safeId(value: string): string { + if (!isSafeIdentifier(value) || SECRET.test(value)) throw new GoalSessionContractError('Provider emitted an unsafe identifier', 'UNSAFE_PROVIDER_VALUE'); + return value; +} + +export function assertSafeProviderIdentifier(value: string): void { + safeId(value); +} + +function safeOptionalId(value: string | undefined): string | undefined { + return value === undefined ? undefined : safeId(value); +} + +function safeOutput(value: string): string { + if (typeof value !== 'string') throw new GoalSessionContractError('Provider emitted non-string output', 'INVALID_PROVIDER_EVENT'); + if (SECRET.test(value) || value.includes('\0')) return '[redacted output]'; + return Buffer.byteLength(value) <= 1024 * 1024 ? value : Buffer.from(value).subarray(0, 1024 * 1024).toString(); +} + +function nonNegativeInteger(value: number | undefined, field: string): number | undefined { + if (value === undefined) return undefined; + if (!Number.isSafeInteger(value) || value < 0) invalidNumeric(field); + return value; +} + +function nonNegativeFinite(value: number | undefined, field: string): number | undefined { + if (value === undefined) return undefined; + if (!Number.isFinite(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) invalidNumeric(field); + return value; +} + +function invalidNumeric(field: string): never { + throw new GoalSessionContractError(`Provider emitted an invalid ${field}`, 'INVALID_PROVIDER_EVENT'); +} + +function safeJson(value: GoalSessionJsonValue | undefined): GoalSessionJsonValue | undefined { + if (value === undefined) return undefined; + if (!value || Array.isArray(value) || typeof value !== 'object') return undefined; + const allowed = new Set(['file', 'line', 'progress', 'count', 'status', 'result', 'code', 'language']); + const result: Record = {}; + for (const [key, nested] of Object.entries(value)) { + if (!allowed.has(key) || (nested !== null && !['string', 'number', 'boolean'].includes(typeof nested))) { + throw new GoalSessionContractError('Provider event data contains an undeclared field', 'INVALID_PROVIDER_EVENT'); + } + if (key === 'file') result[key] = safeRepositoryRelativePath(nested, key); + else if (key === 'line' || key === 'count') result[key] = requiredNonNegativeInteger(nested, key); + else if (key === 'progress') result[key] = boundedProgress(nested); + else if (key === 'status') result[key] = safeClosedScalar(nested, key, ['pending', 'in_progress', 'completed', 'failed', 'cancelled']); + else if (key === 'language') result[key] = safeClosedScalar(nested, key, ['typescript', 'javascript', 'json', 'markdown', 'text', 'shell', 'yaml']); + else if (key === 'code') result[key] = safeClosedScalar(nested, key, ['ok', 'failed', 'skipped', 'cancelled']); + else result[key] = safeResultScalar(nested, key); + } + return result; +} + +export function safeRepositoryRelativePath(value: unknown, field = 'file'): string { + if (typeof value !== 'string' || !value || Buffer.byteLength(value) > 1024 + || hasControl(value) || value.startsWith('/') || WINDOWS_OR_UNC.test(value) + || URI_OR_ENDPOINT.test(value) || COMMAND_LIKE.test(value) || value.includes('\\')) { + throw new GoalSessionContractError(`Provider ${field} is not a safe repository-relative path`, 'UNSAFE_PROVIDER_VALUE'); + } + const segments = value.split('/'); + if (segments.some(segment => !segment || segment === '.' || segment === '..') || segments.join('/') !== value) { + throw new GoalSessionContractError(`Provider ${field} is not normalized`, 'UNSAFE_PROVIDER_VALUE'); + } + return value; +} + +function requiredNonNegativeInteger(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) invalidNumeric(field); + return value; +} + +function boundedProgress(value: unknown): number { + const result = requiredNonNegativeInteger(value, 'progress'); + if (result > 100) invalidNumeric('progress'); + return result; +} + +function safeClosedScalar(value: unknown, field: string, values: readonly string[]): string { + if (typeof value !== 'string' || !values.includes(value)) { + throw new GoalSessionContractError(`Provider event data contains an invalid ${field}`, 'INVALID_PROVIDER_EVENT'); + } + return value; +} + +function safeResultScalar(value: unknown, field: string): string | number | boolean | null { + if (value === null || typeof value === 'boolean') return value; + if (typeof value === 'number') return requiredNonNegativeInteger(value, field); + if (typeof value !== 'string' || !value || value.length > 256 || SECRET.test(value) + || hasControl(value) || URI_OR_ENDPOINT.test(value) || COMMAND_LIKE.test(value) + || value.startsWith('/') || WINDOWS_OR_UNC.test(value) || value.includes('../')) { + throw new GoalSessionContractError('Provider event data contains an unsafe value', 'UNSAFE_PROVIDER_VALUE'); + } + return value; +} + +function hasControl(value: string): boolean { + return [...value].some(character => { + const code = character.charCodeAt(0); + return code < 32 || code === 127; + }); +} + +function closed(value: T, allowed: readonly T[], name: string): T { + if (!allowed.includes(value)) throw new GoalSessionContractError(`Provider emitted an invalid ${name}`, 'INVALID_PROVIDER_EVENT'); + return value; +} diff --git a/packages/core/src/agents/goalSession/sqliteGoalSessionKeys.ts b/packages/core/src/agents/goalSession/sqliteGoalSessionKeys.ts new file mode 100644 index 000000000..327ab4a96 --- /dev/null +++ b/packages/core/src/agents/goalSession/sqliteGoalSessionKeys.ts @@ -0,0 +1,19 @@ +import type { + GoalSessionControlTransition, GoalSessionIdentity, GoalTerminalCommit, +} from './contract.js'; + +export function sqliteGoalScope(identity: GoalSessionIdentity): string { + return `${identity.goalId}\0${identity.sessionId}`; +} + +export function sqliteTransitionKey(value: GoalSessionControlTransition): string { + return JSON.stringify([sqliteGoalScope(value.fence), value.fence.controllerEpoch, + value.turnScoped === true && 'turnId' in value.fence ? value.fence.turnId : null, + value.execution.executionId, value.execution.attemptId, value.transitionId]); +} + +export function sqliteTerminalKey(value: GoalTerminalCommit): string { + return JSON.stringify([value.scope, sqliteGoalScope(value.fence), value.fence.controllerEpoch, + value.scope === 'turn' ? value.fence.turnId : null, + value.execution.executionId, value.execution.attemptId]); +} diff --git a/packages/core/src/agents/goalSession/sqliteGoalSessionSchema.ts b/packages/core/src/agents/goalSession/sqliteGoalSessionSchema.ts new file mode 100644 index 000000000..22fd1c86a --- /dev/null +++ b/packages/core/src/agents/goalSession/sqliteGoalSessionSchema.ts @@ -0,0 +1,65 @@ +import Database from 'better-sqlite3'; +import { GoalSessionContractError } from './errors.js'; + +const REQUIRED_SCHEMA: Readonly> = { + goals: ['goal_id', 'agent', 'effective_model', 'lease_epoch'], + goal_provider_sessions: ['session_id', 'goal_id', 'provider_thread_id', 'lease_generation', 'current_turn_id', 'current_execution_id', 'current_attempt_id'], + goal_event_state: ['goal_id', 'high_watermark', 'projection_sequence'], + goal_events: ['goal_id', 'sequence', 'kind', 'event_type', 'payload_json', 'idempotency_key', 'lease_epoch', 'schema_version', 'payload_bytes'], + goal_messages: ['message_id', 'goal_id', 'sequence', 'queue_ordinal', 'body', 'state', 'claimed_by', 'delivered_at', 'acknowledged_at', 'created_at'], + goal_session_runtime_state: ['session_id', 'goal_id', 'scope', 'payload_json'], + goal_session_runtime_commits: ['session_id', 'goal_id', 'kind', 'identity'], + goal_session_runtime_model_changes: ['session_id', 'goal_id', 'scope', 'operation_id', 'sequence', 'model', 'status', 'acknowledgement_json'], + goal_session_runtime_model_sequences: ['session_id', 'goal_id', 'scope', 'next_sequence'], + goal_session_runtime_provider_effects: ['session_id', 'goal_id', 'scope', 'operation_id', 'kind', 'stage', 'status', 'claim_token', 'outcome_json', 'updated_at'], +}; + +export function assertSqliteGoalControlSchema(database: Database.Database): void { + for (const [table, required] of Object.entries(REQUIRED_SCHEMA)) { + const rows = database.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; + const columns = new Set(rows.map(row => row.name)); + if (required.some(column => !columns.has(column))) throw new GoalSessionContractError( + `Required authoritative control persistence is absent: ${table}`, 'AUTHORITATIVE_DOMAIN_MISSING', + ); + } +} + +export function replayableProviderOutcomeJson(value: unknown): string { + const serialized = JSON.stringify(closedJson(value ?? null, new Set(), 0)); + if (Buffer.byteLength(serialized, 'utf8') > 64 * 1024) throw invalidOutcome(); + return serialized; +} + +function closedJson(value: unknown, seen: Set, depth: number): unknown { + if (depth > 32) throw invalidOutcome(); + if (value === null || typeof value === 'string' || typeof value === 'boolean') return value; + if (typeof value === 'number') { + if (!Number.isFinite(value) || Object.is(value, -0)) throw invalidOutcome(); + return value; + } + if (!value || typeof value !== 'object' || seen.has(value)) throw invalidOutcome(); + seen.add(value); + try { + if (Array.isArray(value)) { + const keys = Object.keys(value); + if (keys.length !== value.length || keys.some((key, index) => key !== String(index))) throw invalidOutcome(); + return value.map(item => closedJson(item, seen, depth + 1)); + } + if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw invalidOutcome(); + if (Object.getOwnPropertySymbols(value).length > 0) throw invalidOutcome(); + const result: Record = {}; + for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(value))) { + if (!descriptor.enumerable || !('value' in descriptor) || !key) throw invalidOutcome(); + result[key] = closedJson(descriptor.value, seen, depth + 1); + } + return result; + } finally { + seen.delete(value); + } +} + +function invalidOutcome(): GoalSessionContractError { + return new GoalSessionContractError( + 'Provider outcome is not bounded lossless JSON', 'PROVIDER_EFFECT_IN_DOUBT', + ); +} diff --git a/packages/core/src/agents/goalSession/supervisedCodexOpenFactory.ts b/packages/core/src/agents/goalSession/supervisedCodexOpenFactory.ts new file mode 100644 index 000000000..721c42710 --- /dev/null +++ b/packages/core/src/agents/goalSession/supervisedCodexOpenFactory.ts @@ -0,0 +1,94 @@ +import type { + GoalPendingCancellationContext, GoalProviderCancelRequest, GoalProviderOpenRequest, + GoalProviderSessionSnapshot, GoalRepositoryIdentity, +} from './contract.js'; +import { openSupervisedCodexAppServer, SUPERVISED_CODEX_MODEL } from './CodexAppServerOpen.js'; +import type { + GoalContainerSupervisor, GoalCredentialMount, +} from './GoalContainerSupervisor.js'; +import { GoalSessionContractError } from './errors.js'; +import type { GoalSupervisedOpenPlan } from './goalSessionOpen.js'; +import { issueGoalSupervisedOpenPlan } from './goalSessionOpen.js'; +import { createProviderProtocolDuplex } from './providerProtocolDuplex.js'; + +export interface SupervisedCodexAppServerFactoryOptions { + repository: GoalRepositoryIdentity; + worktreeFingerprint: string; + image: string; + command?: string[]; + credentialMounts?: ReadonlyArray; + environment?: Record; + maxProtocolQueueBytes?: number; +} + +/** Injectable production composition for claimed container transport and App Server open. */ +export interface GoalProviderOpenFactory { + readonly plan: GoalSupervisedOpenPlan; + open(request: GoalProviderOpenRequest): Promise; + cancelPending(request: GoalProviderCancelRequest, pending: GoalPendingCancellationContext): Promise; +} + +export function createSupervisedCodexAppServerFactory( + containers: GoalContainerSupervisor, + options: SupervisedCodexAppServerFactoryOptions, +): GoalProviderOpenFactory { + const credentialTargets = (options.credentialMounts ?? []).map(mount => mount.target); + const fields: GoalSupervisedOpenPlan = { + repository: options.repository, + requestedModel: SUPERVISED_CODEX_MODEL, + providerHomeTarget: '/home/node/.codex', + credentialTargets, + }; + const plan = issueGoalSupervisedOpenPlan(fields, { + async createTransport(claim) { + const duplex = createProviderProtocolDuplex(options.maxProtocolQueueBytes); + let started: Awaited>; + try { + started = await containers.startOpen({ + goalId: claim.operationFence.goalId, + sessionId: claim.operationFence.sessionId, + controllerEpoch: claim.operationFence.controllerEpoch, + executionId: claim.executionId, + attemptId: claim.attemptId, + deterministicOpenKey: claim.deterministicOpenKey, + operationFence: claim.operationFence, + image: options.image, + command: options.command ?? ['codex', 'app-server'], + worktreePath: options.repository.worktreePath, + worktreeFingerprint: options.worktreeFingerprint, + providerHomeTarget: '/home/node/.codex', + environment: options.environment, + credentialMounts: options.credentialMounts, + outputObserver: duplex.observer, + }); + } catch (error) { + await containers.cancelPendingOpen(claim).catch(() => undefined); + throw error; + } + duplex.bindExecution(started.execution); + return duplex.transport; + }, + async cancelPending(claim) { + await containers.cancelPendingOpen(claim); + }, + transferPending(claim) { + containers.transferPendingOpen(claim); + }, + }); + return { + plan, + async open(request) { + if (!request.openContext) throw new GoalSessionContractError( + 'Claimed Codex App Server context is missing', 'OPEN_CONTEXT_MISSING', + ); + return openSupervisedCodexAppServer(request.openContext, request.persisted); + }, + async cancelPending(request, pending) { + await containers.cancelPendingOpenAttempt({ + goalId: request.goalId, sessionId: request.sessionId, + attemptId: pending.initializationIntent.attemptId, + deterministicOpenKey: pending.initializationIntent.deterministicOpenKey, + }); + }, + }; +} diff --git a/packages/core/src/agents/goalSession/support.ts b/packages/core/src/agents/goalSession/support.ts new file mode 100644 index 000000000..71334152b --- /dev/null +++ b/packages/core/src/agents/goalSession/support.ts @@ -0,0 +1,110 @@ +import type { + DurableCorrectiveMessage, + GoalExecutionIdentity, + GoalProviderSessionSnapshot, + GoalProviderTurnContext, + GoalSessionControlFence, + GoalSessionIdentity, + GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; +import { assertSafeProviderIdentifier } from './securityBoundary.js'; +import { sanitizeRecoveryMetadata } from './recoveryMetadata.js'; +import { isSafeIdentifier } from './safeIdentifier.js'; + +/** Sentinel turn identity used by session-scoped control/audit events. */ +export function controlExecutionIdentity(state: Pick): GoalExecutionIdentity { + return { + executionId: `control-${state.sessionId}`, + attemptId: `epoch-${state.controllerEpoch}`, + }; +} + +export function nowIso(): string { + return new Date().toISOString(); +} + +export function validateIdentity(identity: GoalSessionIdentity): void { + if (!isSafeIdentifier(identity.goalId) || !isSafeIdentifier(identity.sessionId)) { + throw new GoalSessionContractError('goalId and sessionId must be safe opaque identifiers', 'INVALID_IDENTITY'); + } +} + +export function validateEpoch(epoch: number): void { + if (!Number.isSafeInteger(epoch) || epoch < 0) { + throw new GoalSessionContractError('controllerEpoch must be a non-negative safe integer', 'INVALID_EPOCH'); + } +} + +export function validateControlFence(fence: GoalSessionControlFence): void { + validateIdentity(fence); + validateEpoch(fence.controllerEpoch); +} + +export function persistedSnapshot(state: GoalSessionState): GoalProviderSessionSnapshot { + if (!state.providerSessionId || state.recoveryMetadata === undefined) { + throw new GoalSessionContractError( + 'Provider identity/checkpoint is not durable; the session cannot be resumed safely', + 'SESSION_NOT_RECOVERABLE', + ); + } + try { + assertSafeProviderIdentifier(state.providerSessionId); + if (state.currentModel !== undefined) assertSafeProviderIdentifier(state.currentModel); + } catch { + throw new GoalSessionContractError('Durable provider identity contains an unsafe value', 'UNSAFE_PROVIDER_VALUE'); + } + return { + providerSessionId: state.providerSessionId, + recoveryMetadata: sanitizeRecoveryMetadata(state.recoveryMetadata, state.provider), + model: state.currentModel, + }; +} + +export function providerTurnContext(state: GoalSessionState): GoalProviderTurnContext { + if (state.providerSessionId && state.recoveryMetadata !== undefined) { + return { binding: 'bound', snapshot: persistedSnapshot(state) }; + } + if (state.initializationIntent) { + return { binding: 'pending', initializationIntent: { + attemptId: state.initializationIntent.attemptId, + deterministicOpenKey: state.initializationIntent.deterministicOpenKey, + recordedAt: state.initializationIntent.recordedAt, + } }; + } + throw new GoalSessionContractError( + 'The first provider turn has neither a durable native ID nor initialization intent', + 'SESSION_INITIALIZATION_INTENT_MISSING', + ); +} + +export function nextState(state: GoalSessionState, changes: Partial): Omit { + const withoutVersion: Partial = { ...state }; + const effectiveChanges = { ...changes }; + delete withoutVersion.version; + // A committed recovery receipt is valid only until the next durable state + // mutation. Spread-based updates may echo the same object; only an explicit + // new receipt (the atomic recovery transaction) is retained. + delete withoutVersion.completedRecovery; + if (effectiveChanges.completedRecovery === state.completedRecovery) delete effectiveChanges.completedRecovery; + return { ...withoutVersion, ...effectiveChanges, updatedAt: nowIso() } as Omit; +} + +export function assertProviderIdentity(state: GoalSessionState, snapshot: GoalProviderSessionSnapshot): void { + try { + assertSafeProviderIdentifier(snapshot.providerSessionId); + if (snapshot.model !== undefined) assertSafeProviderIdentifier(snapshot.model); + } catch { + throw new GoalSessionContractError('Provider snapshot contains an unsafe identity or model', 'UNSAFE_PROVIDER_VALUE'); + } + if (state.providerSessionId && state.providerSessionId !== snapshot.providerSessionId) { + throw new GoalSessionContractError( + `Provider attempted to replace session "${state.providerSessionId}" with "${snapshot.providerSessionId}"`, + 'PROVIDER_SESSION_CHANGED', + ); + } +} + +export function firstPendingCorrectiveMessage(messages: DurableCorrectiveMessage[]): DurableCorrectiveMessage | undefined { + return [...messages].sort((a, b) => a.sequence - b.sequence)[0]; +} diff --git a/packages/core/src/agents/goalSession/terminalContainerCleanup.ts b/packages/core/src/agents/goalSession/terminalContainerCleanup.ts new file mode 100644 index 000000000..8fbe6c9f2 --- /dev/null +++ b/packages/core/src/agents/goalSession/terminalContainerCleanup.ts @@ -0,0 +1,28 @@ +import { realpath, rm } from 'node:fs/promises'; +import path from 'node:path'; +import type { GoalContainerLayout, GoalContainerRetentionPolicy } from './goalContainerLayout.js'; +import { GOAL_SCOPE_PATTERN } from './goalContainerLayout.js'; + +export async function cleanTerminalGoalSession( + options: { + baseDirectory: string; retention: GoalContainerRetentionPolicy; layout: GoalContainerLayout; + terminalAt: Date; outcome: 'succeeded' | 'cancelled' | 'failed'; currentTime: Date; + }, +): Promise { + const { baseDirectory, retention, layout, terminalAt, outcome, currentTime } = options; + const duration = outcome === 'succeeded' ? retention.succeededMs + : outcome === 'cancelled' ? retention.cancelledMs : retention.failedMs; + if (currentTime < new Date(terminalAt.getTime() + duration)) return false; + const realGoals = await realpath(path.join(await realpath(baseDirectory), 'goals')).catch(() => null); + if (!realGoals) return false; + const lexicalRoot = path.resolve(layout.sessionRoot); + if (path.dirname(lexicalRoot) !== realGoals || !GOAL_SCOPE_PATTERN.test(path.basename(lexicalRoot))) { + throw new Error('Refusing to clean a path outside the goal container resource directory'); + } + let resolvedRoot: string; + try { resolvedRoot = await realpath(lexicalRoot); } + catch { return false; } + if (resolvedRoot !== lexicalRoot) throw new Error('Refusing to clean a symlinked goal session directory'); + await rm(resolvedRoot, { recursive: true, force: true }); + return true; +} diff --git a/packages/core/src/agents/goalSession/turnCompletionProtocol.ts b/packages/core/src/agents/goalSession/turnCompletionProtocol.ts new file mode 100644 index 000000000..29013160c --- /dev/null +++ b/packages/core/src/agents/goalSession/turnCompletionProtocol.ts @@ -0,0 +1,22 @@ +import type { GoalSessionEvent, GoalSessionState } from './contract.js'; + +export function completesAtAfterTurnPause( + state: GoalSessionState, + outcome: Extract['outcome'], + pauseCapability: 'active_turn' | 'after_turn', +): boolean { + return outcome === 'succeeded' + && pauseCapability === 'after_turn' + && (state.status === 'pause_requested' + || state.status === 'paused' + || state.pendingAfterTurnPause === true); +} + +export function needsAfterTurnPauseAudit( + state: GoalSessionState, + outcome: Extract['outcome'], + pauseCapability: 'active_turn' | 'after_turn', +): boolean { + return completesAtAfterTurnPause(state, outcome, pauseCapability) + && (state.status === 'pause_requested' || state.pendingAfterTurnPause === true); +} diff --git a/packages/core/src/agents/goalSession/turnDelivery.ts b/packages/core/src/agents/goalSession/turnDelivery.ts new file mode 100644 index 000000000..faef19051 --- /dev/null +++ b/packages/core/src/agents/goalSession/turnDelivery.ts @@ -0,0 +1,28 @@ +import type { GoalExecutionIdentity, GoalSessionState } from './contract.js'; + +export type RunGoalTurnResult = + | { disposition: 'started'; state: GoalSessionState; execution: GoalExecutionIdentity } + /** A redelivery observes durable state and never invokes or completes the provider itself. */ + | { disposition: 'duplicate'; reattached: boolean; state: GoalSessionState; execution: GoalExecutionIdentity }; + +export function duplicateTurnResult( + state: GoalSessionState, + turnId: string, + fallback: GoalExecutionIdentity, +): RunGoalTurnResult | undefined { + if (state.activeTurn?.turnId === turnId) { + const execution = { executionId: state.activeTurn.executionId, attemptId: state.activeTurn.attemptId }; + return { disposition: 'duplicate', reattached: true, state, execution }; + } + if (!state.completedTurnIds.includes(turnId)) return undefined; + const recorded = state.completedTurns?.find(turn => turn.turnId === turnId); + if (recorded) { + return { + disposition: 'duplicate', + reattached: true, + state, + execution: { executionId: recorded.executionId, attemptId: recorded.attemptId }, + }; + } + return { disposition: 'duplicate', reattached: false, state, execution: fallback }; +} diff --git a/packages/core/src/agents/goalSession/turnExecutionProtocol.ts b/packages/core/src/agents/goalSession/turnExecutionProtocol.ts new file mode 100644 index 000000000..d4ba59048 --- /dev/null +++ b/packages/core/src/agents/goalSession/turnExecutionProtocol.ts @@ -0,0 +1,49 @@ +import type { GoalBeginTurnRequest, GoalExecutionIdentity, GoalSessionState } from './contract.js'; + +export function turnExecution( + state: GoalSessionState, + request: { turnId: string; executionId: string; attemptId?: string }, + mint: () => string, + mintFresh: (previous: string) => string, +): GoalExecutionIdentity { + const retry = state.retryTurn?.turnId === request.turnId + && state.retryTurn.executionId === request.executionId ? state.retryTurn : undefined; + return { + executionId: request.executionId, + attemptId: retry ? mintFresh(retry.crashedAttemptId) : request.attemptId ?? mint(), + }; +} + +export function resolveDeferredModel( + state: GoalSessionState, + fallback: string, + enabled: boolean, +): { + requestedModel: string; + activeModelChange?: NonNullable['modelChange']; + providerModelChange?: GoalBeginTurnRequest['modelChange']; +} { + const requestedModel = state.pendingModelChange ?? state.modelChangeIntent?.model ?? fallback; + const modelIntent = enabled && state.pendingModelChange === requestedModel + && state.modelChangeIntent?.model === requestedModel ? state.modelChangeIntent : undefined; + const generation = modelIntent?.generation ?? state.modelChangeGeneration ?? 0; + return { + requestedModel, + activeModelChange: modelIntent ? { + modelChangeId: modelIntent.modelChangeId, generation, + previousModel: modelIntent.previousModel ?? state.currentModel, + } : undefined, + providerModelChange: modelIntent ? { modelChangeId: modelIntent.modelChangeId, generation } : undefined, + }; +} + +export function settledResumeKind( + state: GoalSessionState, + kind: 'active_turn' | 'recovered_after_turn', +): boolean { + const liveStatus = kind === 'active_turn' ? state.status === 'running' + : state.status === 'running' || state.status === 'pause_requested'; + return liveStatus && Boolean(state.activeTurn) && state.resumeIntent?.phase === 'settled' + && state.completedResume?.operationId === state.resumeIntent.operationId + && state.completedResume.kind === kind; +} diff --git a/packages/core/src/agents/goalSession/turnStreamProtocol.ts b/packages/core/src/agents/goalSession/turnStreamProtocol.ts new file mode 100644 index 000000000..b61b2c412 --- /dev/null +++ b/packages/core/src/agents/goalSession/turnStreamProtocol.ts @@ -0,0 +1,81 @@ +import { createHash } from 'node:crypto'; +import type { + GoalExecutionIdentity, + GoalNativeSessionIdTiming, + GoalSessionFence, + GoalSessionEvent, + GoalSessionState, +} from './contract.js'; +import { GoalSessionContractError } from './errors.js'; + +export function assertFirstTurnIdentityEvent( + state: GoalSessionState, + event: GoalSessionEvent, + idTiming: GoalNativeSessionIdTiming, +): void { + if (state.providerSessionId || idTiming !== 'first_turn') return; + if (event.type !== 'checkpoint' || !event.providerSessionId?.trim()) { + throw new GoalSessionContractError( + 'A first-turn provider must durably bind its native session ID before emitting authoritative work', + 'FIRST_TURN_ID_NOT_BOUND', + ); + } +} + +export function assertSuppliedMessagesAcknowledged( + event: GoalSessionEvent, + awaitingMessageIds: string[], +): void { + if (event.type !== 'completion' || event.outcome !== 'succeeded' || awaitingMessageIds.length === 0) return; + throw new GoalSessionContractError( + `Provider reported success without acknowledging supplied corrective message "${awaitingMessageIds[0]}"`, + 'MESSAGE_ACK_MISSING', + ); +} + +export function isAtomicTurnAudit(event: GoalSessionEvent): boolean { + return event.type === 'model_changed' || event.type === 'pause_boundary' || event.type === 'usage'; +} + +export function streamAuditTransitionId( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + event: Extract, +): string { + const occurrence = streamTransitionOccurrence(event); + const digest = createHash('sha256') + .update(JSON.stringify([ + fence.goalId, + fence.sessionId, + fence.controllerEpoch, + fence.turnId, + execution.executionId, + execution.attemptId, + occurrence, + ])) + .digest('hex') + .slice(0, 32); + return `stream-audit-${digest}`; +} + +/** Validates and selects the provider-stable occurrence identity. IDs win over ordinals. */ +export function streamTransitionOccurrence( + event: Extract, +): readonly ['provider_event_id', string] | readonly ['provider_event_ordinal', number] { + if (event.providerEventId !== undefined) { + if (typeof event.providerEventId !== 'string' || !event.providerEventId.trim()) { + throw new GoalSessionContractError( + 'Streamed model/pause transition providerEventId must be non-empty', + 'STREAM_TRANSITION_ID_INVALID', + ); + } + return ['provider_event_id', event.providerEventId]; + } + if (Number.isSafeInteger(event.providerEventOrdinal) && (event.providerEventOrdinal ?? -1) >= 0) { + return ['provider_event_ordinal', event.providerEventOrdinal as number]; + } + throw new GoalSessionContractError( + 'Streamed model/pause transition requires a stable providerEventId or providerEventOrdinal', + 'STREAM_TRANSITION_ID_MISSING', + ); +} diff --git a/packages/core/src/agents/goalSession/worktreeIdentity.ts b/packages/core/src/agents/goalSession/worktreeIdentity.ts new file mode 100644 index 000000000..206f73523 --- /dev/null +++ b/packages/core/src/agents/goalSession/worktreeIdentity.ts @@ -0,0 +1,117 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import { realpath } from 'node:fs/promises'; +import type { GoalRepositoryIdentity } from './contract.js'; + +const SAFE_REMOTE_PROTOCOLS = new Set(['git:', 'http:', 'https:', 'ssh:']); +const SAFE_HOST = /^(?:[a-z\d](?:[a-z\d-]*[a-z\d])?)(?:\.(?:[a-z\d](?:[a-z\d-]*[a-z\d])?))+$/i; +const SAFE_PATH_SEGMENT = /^[a-z\d._~-]+$/i; +const SAFE_BRANCH = /^(?![./])(?!.*(?:\.\.|@\{|\\|\s|[~^:?*]|\[))(?!.*\.$)[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$/; +const SAFE_SHA = /^[a-f\d]{4,64}$/i; +const SENSITIVE_WORKTREE_ROOTS = [ + '/', '/boot', '/dev', '/etc', '/home', '/proc', '/root', '/run', '/sys', + '/var/run', '/var/lib/docker', '/var/lib/containers', '/var/lib/podman', '/var/lib/containerd', +]; +const SECRET_LIKE = /(?:gh[oprsu]_|github_pat_|sk-|AKIA|bearer[._-]?[A-Za-z0-9]|(?:secret|token|password)[._:-][A-Za-z0-9_-]{6,})/i; + +function normalizedRemoteHost(host: string): string | undefined { + const normalized = host.toLowerCase().replace(/\.$/, ''); + return SAFE_HOST.test(normalized) ? normalized : undefined; +} + +function normalizedHostPath(host: string, repositoryPath: string): string | undefined { + const normalizedHost = normalizedRemoteHost(host); + const cleaned = cleanPath(repositoryPath); + if (!normalizedHost || !cleaned) return undefined; + return (normalizedHost === 'github.com' ? cleaned : `${normalizedHost}/${cleaned}`).toLowerCase(); +} + +function cleanPath(value: string): string | undefined { + const withoutSuffix = value.split(/[?#]/, 1)[0]; + const segments = withoutSuffix.replace(/^\/+|\/+$/g, '').split('/'); + if (segments.length < 2 || segments.some(segment => + !segment || segment === '.' || segment === '..' || !SAFE_PATH_SEGMENT.test(segment))) return undefined; + segments[segments.length - 1] = segments.at(-1)!.replace(/\.git$/i, ''); + if (!segments.at(-1)) return undefined; + return segments.join('/'); +} + +/** + * Converts a Git remote or logical repository name to the credential-free + * identity used for fencing. Userinfo, query strings, and fragments are never + * returned. Undefined means no trustworthy host/path identity was available. + */ +export function normalizeGitRepositoryIdentity(value: string): string | undefined { + const trimmed = value.trim().replace(/^git\+/, ''); + if (!trimmed || [...trimmed].some(character => { + const code = character.charCodeAt(0); + return code < 32 || code === 127; + })) return undefined; + if (trimmed.includes('://')) { + try { + const remote = new URL(trimmed); + if (!SAFE_REMOTE_PROTOCOLS.has(remote.protocol)) return undefined; + const credentialFreeSshUser = remote.protocol === 'ssh:' + && remote.username.toLowerCase() === 'git' && !remote.password; + if ((remote.username && !credentialFreeSshUser) || remote.password || remote.search || remote.hash) return undefined; + return normalizedHostPath(remote.hostname, remote.pathname); + } catch { + return undefined; + } + } + const scp = /^(?:(.+)@)?([^/:\s]+):\/?(.+)$/.exec(trimmed); + if (scp) { + if (scp[1] && scp[1].toLowerCase() !== 'git') return undefined; + return normalizedHostPath(scp[2], scp[3]); + } + if (trimmed.includes('@') || trimmed.includes(':') || trimmed.includes('\\')) return undefined; + const logical = cleanPath(trimmed); + return logical?.toLowerCase(); +} + +export function normalizeGoalRepositoryIdentity( + repository: GoalRepositoryIdentity, +): GoalRepositoryIdentity | undefined { + const normalized = normalizeGitRepositoryIdentity(repository.repository); + const worktreePath = path.resolve(repository.worktreePath); + const branch = repository.branch.trim(); + if (!normalized || SECRET_LIKE.test(normalized) || repository.worktreePath !== worktreePath + || isSensitiveWorktreePath(worktreePath) || SECRET_LIKE.test(worktreePath) + || !SAFE_BRANCH.test(branch) || SECRET_LIKE.test(branch)) return undefined; + return { + repository: normalized, + worktreePath, + branch, + headSha: SAFE_SHA.test(repository.headSha ?? '') ? repository.headSha!.toLowerCase() : undefined, + }; +} + +/** Canonical ingress validator shared by turns, durable recovery, and Git inspection. */ +export async function normalizeCanonicalGoalRepositoryIdentity( + repository: GoalRepositoryIdentity, +): Promise { + const normalized = normalizeGoalRepositoryIdentity(repository); + if (!normalized) return undefined; + const resolved = await realpath(normalized.worktreePath).catch(() => normalized.worktreePath); + if (resolved !== normalized.worktreePath || isSensitiveWorktreePath(resolved)) return undefined; + return normalized; +} + +export function isSensitiveWorktreePath(value: string): boolean { + const candidate = path.resolve(value); + return SENSITIVE_WORKTREE_ROOTS.some(root => candidate === root || (root !== '/' && candidate.startsWith(`${root}/`))); +} + +/** Shared source policy for worktrees, Git inspection, and credential mounts. */ +export const isSensitiveHostSourcePath = isSensitiveWorktreePath; + +/** Stable logical checkout identity. Mutable HEAD/checkpoint state is deliberately excluded. */ +export function fingerprintGoalWorktree(repository: GoalRepositoryIdentity): string { + const repositoryName = normalizeGitRepositoryIdentity(repository.repository); + if (!repositoryName) throw new Error('Repository identity is not a trustworthy Git repository name or remote'); + return createHash('sha256').update([ + repositoryName, + path.resolve(repository.worktreePath), + repository.branch, + ].join('\0')).digest('hex'); +} diff --git a/packages/core/src/agents/index.ts b/packages/core/src/agents/index.ts new file mode 100644 index 000000000..cc1eb4e88 --- /dev/null +++ b/packages/core/src/agents/index.ts @@ -0,0 +1,2 @@ +export * from './goalSession/index.js'; +export * from './version/index.js'; diff --git a/packages/core/src/claude/docker/dockerExecutor.ts b/packages/core/src/claude/docker/dockerExecutor.ts index 41698d18a..7ebdf88cd 100644 --- a/packages/core/src/claude/docker/dockerExecutor.ts +++ b/packages/core/src/claude/docker/dockerExecutor.ts @@ -16,8 +16,19 @@ import { scheduleForceKill, setupAbortChecker, } from './dockerAbortController.js'; +import { resolveDockerPath, stripAnsiCodes } from './dockerProcessUtils.js'; export { stopDockerContainer } from './dockerContainerControl.js'; +export { + executeSupervisedDockerCommand, + addGoalFenceLabels, +} from './supervisedDockerExecutor.js'; +export type { + SupervisedDockerExecution, + SupervisedDockerFence, + SupervisedDockerOptions, + SupervisedDockerOutput, +} from './supervisedDockerExecutor.js'; export { addTaskAttemptLabelsToDockerArgs, ExecutionAbortedError, @@ -46,6 +57,7 @@ export interface ExecutionResult { export interface RunningTaskContainer { id: string; name: string; } export type LegacyTaskContainerLiveness = 'running' | 'not_found' | 'unavailable'; + export interface DockerCommandOptions { timeout?: number; cwd?: string; worktreePath?: string; stdinData?: string; taskId?: string; streamToRedis?: boolean; streamStderrToRedis?: boolean; stripAnsi?: boolean; /** Resolve with buffered output on timeout so implementation jobs can publish partial work. */ @@ -58,23 +70,6 @@ export interface DockerCommandOptions { interface JsonLineMessage { type?: string; message?: { id?: string; model?: string; }; session_id?: string; conversation_id?: string; } -// ANSI escape code regex for stripping terminal formatting (constructed dynamically to avoid control char lint errors) -const ANSI_REGEX = new RegExp('[' + String.fromCharCode(0x1b) + String.fromCharCode(0x9b) + '][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]', 'g'); - -function stripAnsiCodes(text: string): string { - return text.replace(ANSI_REGEX, ''); -} - -function resolveDockerPath(command: string): string { - if (command !== 'docker') return command; - const paths = ['/usr/bin/docker', '/usr/local/bin/docker', '/bin/docker']; - for (const p of paths) { - try { if (fs.existsSync(p)) { fs.accessSync(p, fs.constants.X_OK); logger.debug({ dockerPath: p }, 'Found docker executable'); return p; } } catch { /* continue */ } - } - logger.debug('Using docker from PATH'); - return 'docker'; -} - /** * Finds an agent container in any lifecycle state by its exact task label and, * when supplied, its attempt-generation label. Name suffixes are intentionally excluded: diff --git a/packages/core/src/claude/docker/dockerProcessUtils.ts b/packages/core/src/claude/docker/dockerProcessUtils.ts new file mode 100644 index 000000000..54a8be29f --- /dev/null +++ b/packages/core/src/claude/docker/dockerProcessUtils.ts @@ -0,0 +1,19 @@ +import fs from 'fs'; +import logger from '../../utils/logger.js'; + +// ANSI escape code regex for stripping terminal formatting (constructed dynamically to avoid control char lint errors) +const ANSI_REGEX = new RegExp('[' + String.fromCharCode(0x1b) + String.fromCharCode(0x9b) + '][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]', 'g'); + +export function stripAnsiCodes(text: string): string { + return text.replace(ANSI_REGEX, ''); +} + +export function resolveDockerPath(command: string): string { + if (command !== 'docker') return command; + const paths = ['/usr/bin/docker', '/usr/local/bin/docker', '/bin/docker']; + for (const p of paths) { + try { if (fs.existsSync(p)) { fs.accessSync(p, fs.constants.X_OK); logger.debug({ dockerPath: p }, 'Found docker executable'); return p; } } catch { /* continue */ } + } + logger.debug('Using docker from PATH'); + return 'docker'; +} diff --git a/packages/core/src/claude/docker/supervisedDockerExecutor.ts b/packages/core/src/claude/docker/supervisedDockerExecutor.ts new file mode 100644 index 000000000..5f2b2bba9 --- /dev/null +++ b/packages/core/src/claude/docker/supervisedDockerExecutor.ts @@ -0,0 +1,412 @@ +import { spawn } from 'child_process'; +import fs from 'fs'; +import type { Readable } from 'stream'; +import { StringDecoder } from 'node:string_decoder'; +import { + abortSpawnedExecution, + createDockerExecutionState, + ExecutionAbortedError, + getExecutionAbortError, + getDockerRunContainerName, + getExecutionOwnershipContext, + resolveExecutionArgs, +} from './dockerExecutionOwnership.js'; +import { scheduleForceKill } from './dockerAbortController.js'; +import { resolveDockerPath } from './dockerProcessUtils.js'; + +export interface SupervisedDockerFence { + goalId: string; + sessionId: string; + controllerEpoch: number; + turnId?: string; + /** Control-scoped eager open deliberately has no turnId. */ + scope?: 'turn' | 'open'; + openKey?: string; + executionId: string; + attemptId: string; + worktreeFingerprint: string; + operationGeneration: number; + operationKind: 'open' | 'turn' | 'resume' | 'reconcile' | 'steer' | 'model' | 'pause' | 'cancel'; + operationId: string; + operationLeaseExpiresAt?: string; +} + +export interface SupervisedDockerOutput extends SupervisedDockerFence { + /** Monotonic arrival order across stdout and stderr for this invocation. */ + sequence: number; + recordedAt: string; + channel: 'stdout' | 'stderr'; + data: string; +} + +export interface SupervisedDockerOptions extends SupervisedDockerFence { + taskId?: string; + cwd?: string; + signal?: AbortSignal; + timeout?: number; + /** + * Allow-listed environment injected into the spawned docker client's own + * environment. Callers pair this with `docker run --env NAME` (name only) so + * secret values are never placed in argv/process listings. + */ + env?: Record; + /** Largest single durable chunk; larger stream reads are split. Default 64 KiB. */ + maxChunkBytes?: number; + /** Hard bound on buffered-but-undelivered bytes before overflow cancellation. Default 8 MiB. */ + maxQueuedBytes?: number; + /** Called once per arriving stream chunk. Delivery is serialized and back-pressured. */ + durableOutput: (output: SupervisedDockerOutput) => void | Promise; +} + +export interface SupervisedDockerExecution { + containerName: string | null; + writeInput(data: string): Promise; + closeInput(): void; + /** Terminal cancellation is immediate and deliberately separate from provider pause. */ + cancel(reason?: Error): Promise; + completion: Promise<{ exitCode: number | null }>; +} + +const DEFAULT_MAX_CHUNK_BYTES = 64 * 1024; +const DEFAULT_MAX_QUEUED_BYTES = 8 * 1024 * 1024; + +function assertPositiveSafeInteger(value: number | undefined, name: string): void { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`Supervised Docker ${name} must be a positive safe integer`); + } +} + +/** + * Resolves the backpressure limits into a coherent, positive, safe-integer + * policy. A caller may set only the queued bound; the per-chunk default is then + * clamped so it can never exceed it. Explicitly incoherent overrides are rejected. + */ +function resolveBackpressureLimits(options: SupervisedDockerOptions): { maxChunkBytes: number; maxQueuedBytes: number } { + assertPositiveSafeInteger(options.maxQueuedBytes, 'maxQueuedBytes'); + assertPositiveSafeInteger(options.maxChunkBytes, 'maxChunkBytes'); + const maxQueuedBytes = options.maxQueuedBytes ?? DEFAULT_MAX_QUEUED_BYTES; + const maxChunkBytes = options.maxChunkBytes ?? Math.min(DEFAULT_MAX_CHUNK_BYTES, maxQueuedBytes); + if (maxChunkBytes > maxQueuedBytes) { + throw new Error('Supervised Docker maxChunkBytes must not exceed maxQueuedBytes'); + } + return { maxChunkBytes, maxQueuedBytes }; +} + +function isUtf8ContinuationByte(byte: number): boolean { + return (byte & 0xc0) === 0x80; +} + +/** + * Splits a buffer into slices no larger than maxChunkBytes, backing each split + * off any trailing UTF-8 continuation bytes so a multi-byte character is never + * cut across a chunk boundary and every slice decodes to valid text. + */ +function splitBuffer(buffer: Buffer, maxChunkBytes: number): Buffer[] { + if (buffer.length <= maxChunkBytes) return [buffer]; + const slices: Buffer[] = []; + let offset = 0; + while (offset < buffer.length) { + let end = Math.min(offset + maxChunkBytes, buffer.length); + if (end < buffer.length) { + while (end > offset && isUtf8ContinuationByte(buffer[end])) end -= 1; + // A run of continuation bytes longer than a chunk (only possible for + // malformed input) falls back to a hard split to guarantee progress. + if (end === offset) end = Math.min(offset + maxChunkBytes, buffer.length); + } + slices.push(buffer.subarray(offset, end)); + offset = end; + } + return slices; +} + +/** + * Serializes stream chunks to a durable sink while bounding memory. Chunks are + * delivered strictly in arrival order across stdout and stderr. When buffered + * bytes cross a high-water mark the source streams are paused, and they resume + * once the sink drains below the low-water mark, so a slow sink cannot cause + * unbounded buffering. Exceeding the hard cap raises an actionable overflow error. + */ +interface OrderedBackpressureSinkConfig { + base: SupervisedDockerFence; + deliver: (output: SupervisedDockerOutput) => void | Promise; + streams: () => Array; + onOverflow: (error: Error) => void; + maxChunkBytes: number; + maxQueuedBytes: number; +} + +class OrderedBackpressureSink { + private readonly queue: SupervisedDockerOutput[] = []; + private queuedBytes = 0; + private draining = false; + private paused = false; + private failed = false; + private nextSequence = 1; + private loop: Promise = Promise.resolve(); + private readonly base: SupervisedDockerFence; + private readonly deliver: (output: SupervisedDockerOutput) => void | Promise; + private readonly streams: () => Array; + private readonly onOverflow: (error: Error) => void; + private readonly maxChunkBytes: number; + private readonly maxQueuedBytes: number; + private readonly highWaterMark: number; + private readonly lowWaterMark: number; + + constructor(config: OrderedBackpressureSinkConfig) { + this.base = config.base; + this.deliver = config.deliver; + this.streams = config.streams; + this.onOverflow = config.onOverflow; + this.maxChunkBytes = config.maxChunkBytes; + this.maxQueuedBytes = config.maxQueuedBytes; + this.highWaterMark = Math.max(1, Math.floor(config.maxQueuedBytes / 2)); + this.lowWaterMark = Math.max(1, Math.floor(config.maxQueuedBytes / 4)); + } + + enqueue(channel: 'stdout' | 'stderr', buffer: Buffer): void { + if (this.failed) return; + for (const slice of splitBuffer(buffer, this.maxChunkBytes)) { + this.queue.push({ + goalId: this.base.goalId, + sessionId: this.base.sessionId, + controllerEpoch: this.base.controllerEpoch, + turnId: this.base.turnId, + executionId: this.base.executionId, + attemptId: this.base.attemptId, + worktreeFingerprint: this.base.worktreeFingerprint, + operationGeneration: this.base.operationGeneration, + operationKind: this.base.operationKind, + operationId: this.base.operationId, + operationLeaseExpiresAt: this.base.operationLeaseExpiresAt, + sequence: this.nextSequence, + recordedAt: new Date().toISOString(), + channel, + data: slice.toString(), + }); + this.nextSequence += 1; + this.queuedBytes += slice.length; + if (this.queuedBytes >= this.highWaterMark) this.setPaused(true); + // Enforce the hard cap while enqueuing each slice so a single + // oversized read is stopped mid-split instead of being fully + // buffered before the bound is checked. + if (this.queuedBytes > this.maxQueuedBytes) { + this.fail(new Error(`Supervised Docker output exceeded the ${this.maxQueuedBytes}-byte backpressure bound; the durable sink is too slow to keep up`)); + return; + } + } + this.ensureDraining(); + } + + /** Resolves once every queued chunk has been delivered (or delivery failed). */ + settle(): Promise { + return this.loop; + } + + private ensureDraining(): void { + if (this.draining || this.failed) return; + this.draining = true; + this.loop = this.drain(); + } + + private async drain(): Promise { + while (this.queue.length && !this.failed) { + const item = this.queue.shift()!; + try { + await this.deliver(item); + } catch (error) { + this.fail(error instanceof Error ? error : new Error(String(error))); + return; + } + this.queuedBytes -= Buffer.byteLength(item.data); + if (this.paused && this.queuedBytes <= this.lowWaterMark) this.setPaused(false); + } + this.draining = false; + if (this.paused && this.queuedBytes <= this.lowWaterMark) this.setPaused(false); + } + + private setPaused(paused: boolean): void { + if (this.paused === paused) return; + this.paused = paused; + for (const stream of this.streams()) { + if (paused) stream?.pause(); + else stream?.resume(); + } + } + + private fail(error: Error): void { + if (this.failed) return; + this.failed = true; + this.queue.length = 0; + this.queuedBytes = 0; + this.setPaused(false); + this.onOverflow(error); + } +} + +export function addGoalFenceLabels(args: string[], fence: SupervisedDockerFence): string[] { + if (args[0] !== 'run') return args; + return [ + 'run', + '--label', `propr.goal.id=${fence.goalId}`, + '--label', `propr.goal.session=${fence.sessionId}`, + '--label', `propr.goal.controller-epoch=${fence.controllerEpoch}`, + '--label', `propr.goal.scope=${fence.scope ?? 'turn'}`, + ...(fence.turnId ? ['--label', `propr.goal.turn=${fence.turnId}`] : []), + ...(fence.openKey ? ['--label', `propr.goal.open-key=${fence.openKey}`] : []), + '--label', `propr.goal.execution=${fence.executionId}`, + '--label', `propr.goal.attempt=${fence.attemptId}`, + '--label', `propr.goal.worktree-fingerprint=${fence.worktreeFingerprint}`, + '--label', `propr.goal.operation-generation=${fence.operationGeneration}`, + '--label', `propr.goal.operation-kind=${fence.operationKind}`, + '--label', `propr.goal.operation-id=${fence.operationId}`, + ...(fence.operationLeaseExpiresAt + ? ['--label', `propr.goal.operation-lease-expires-at=${fence.operationLeaseExpiresAt}`] : []), + ...args.slice(1), + ]; +} + +function validateSupervisedOptions(args: string[], options: SupervisedDockerOptions): void { + if (args[0] !== 'run') throw new Error('Supervised Docker execution only supports docker run'); + if (!options.goalId || !options.sessionId || !options.executionId || !options.attemptId + || !options.worktreeFingerprint || !Number.isSafeInteger(options.controllerEpoch) + || !Number.isSafeInteger(options.operationGeneration) || !options.operationKind || !options.operationId) { + throw new Error('A valid goal/session/controller epoch/turn fence is required'); + } + if ((options.scope ?? 'turn') === 'turn' && !options.turnId) { + throw new Error('A turn-scoped supervised Docker execution requires turnId'); + } + if (options.scope === 'open' && (options.turnId !== undefined || !options.openKey)) { + throw new Error('A control-scoped supervised Docker open requires openKey and forbids turnId'); + } + if (options.timeout !== undefined && (!Number.isSafeInteger(options.timeout) || options.timeout <= 0)) { + throw new Error('Supervised Docker timeout must be a positive safe integer'); + } +} + +/** + * Starts a controlled duplex Docker invocation for a goal turn. Unlike the + * legacy one-shot executor, stdin remains open and output deltas are awaited by + * an injected durable sink with explicit backpressure. No expiring full-output + * snapshot is maintained. + */ +export function executeSupervisedDockerCommand( + args: string[], + options: SupervisedDockerOptions, +): SupervisedDockerExecution { + validateSupervisedOptions(args, options); + const backpressureLimits = resolveBackpressureLimits(options); + const ownershipContext = getExecutionOwnershipContext(); + const executionSignal = options.signal ?? ownershipContext?.signal; + const initialAbortError = getExecutionAbortError(executionSignal); + if (initialAbortError) throw initialAbortError; + const fencedArgs = addGoalFenceLabels( + resolveExecutionArgs('docker', args, options.taskId, ownershipContext?.attemptGeneration), + options, + ); + const containerName = getDockerRunContainerName(fencedArgs); + const child = spawn(resolveDockerPath('docker'), fencedArgs, { + stdio: ['pipe', 'pipe', 'pipe'], + cwd: options.cwd && fs.existsSync(options.cwd) ? options.cwd : undefined, + // Exact allowlist: never inherit host-controlled Docker, loader, SSH, or + // credential variables into this security boundary. + env: options.env ?? {}, + }); + const state = createDockerExecutionState(); + let outputFailure: unknown; + let timeoutHandle: ReturnType | undefined; + let cancelReason: Error | undefined; + let settled = false; + let settleCompletion: ((result: { exitCode: number | null }) => void) | undefined; + let rejectCompletion: ((error: unknown) => void) | undefined; + const completion = new Promise<{ exitCode: number | null }>((resolve, reject) => { + settleCompletion = resolve; + rejectCompletion = reject; + }); + const cancel = async (reason = new ExecutionAbortedError()): Promise => { + if (settled) return; + cancelReason ??= reason; + await abortSpawnedExecution(child, state, { + namedContainer: containerName, + scheduleForceKill, + taskId: options.taskId, + attemptGeneration: ownershipContext?.attemptGeneration, + }); + }; + const sink = new OrderedBackpressureSink({ + // Runtime options also carry environment, host paths, callbacks, and + // process-control fields. Build the public output fence explicitly so + // structural excess properties can never cross the delivery boundary. + base: { + goalId: options.goalId, + sessionId: options.sessionId, + controllerEpoch: options.controllerEpoch, + turnId: options.turnId, + executionId: options.executionId, + attemptId: options.attemptId, + worktreeFingerprint: options.worktreeFingerprint, + operationGeneration: options.operationGeneration, + operationKind: options.operationKind, + operationId: options.operationId, + operationLeaseExpiresAt: options.operationLeaseExpiresAt, + }, + deliver: options.durableOutput, + streams: () => [child.stdout, child.stderr], + onOverflow: error => { outputFailure ??= error; void cancel(error); }, + maxChunkBytes: backpressureLimits.maxChunkBytes, + maxQueuedBytes: backpressureLimits.maxQueuedBytes, + }); + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + child.stdout?.on('data', (data: Buffer) => { + const decoded = stdoutDecoder.write(data); + if (decoded) sink.enqueue('stdout', Buffer.from(decoded)); + }); + child.stderr?.on('data', (data: Buffer) => { + const decoded = stderrDecoder.write(data); + if (decoded) sink.enqueue('stderr', Buffer.from(decoded)); + }); + const abortListener = (): void => { void cancel(getExecutionAbortError(executionSignal) ?? undefined); }; + executionSignal?.addEventListener('abort', abortListener, { once: true }); + if (options.timeout !== undefined) { + timeoutHandle = setTimeout(() => { void cancel(new Error(`Supervised Docker command timed out after ${options.timeout}ms`)); }, options.timeout); + } + child.once('close', (exitCode: number | null) => { + const finalStdout = stdoutDecoder.end(); + const finalStderr = stderrDecoder.end(); + if (finalStdout) sink.enqueue('stdout', Buffer.from(finalStdout)); + if (finalStderr) sink.enqueue('stderr', Buffer.from(finalStderr)); + settled = true; + if (timeoutHandle) clearTimeout(timeoutHandle); + executionSignal?.removeEventListener('abort', abortListener); + // On a sink failure (e.g. overflow) the delivery may be stuck, so don't + // wait on it; otherwise drain in order before settling. + const drained = outputFailure ? Promise.resolve() : sink.settle(); + void drained.then(async () => { + if (state.teardownPromise) await state.teardownPromise; + if (outputFailure) rejectCompletion?.(outputFailure); + else if (cancelReason) rejectCompletion?.(cancelReason); + else settleCompletion?.({ exitCode }); + }); + }); + child.once('error', error => { + settled = true; + if (timeoutHandle) clearTimeout(timeoutHandle); + executionSignal?.removeEventListener('abort', abortListener); + rejectCompletion?.(error); + }); + + return { + containerName, + writeInput(data: string): Promise { + if (!child.stdin || child.stdin.destroyed || child.stdin.writableEnded) { + return Promise.reject(new Error('Supervised Docker stdin is closed')); + } + return new Promise((resolve, reject) => { + child.stdin!.write(data, error => error ? reject(error) : resolve()); + }); + }, + closeInput(): void { child.stdin?.end(); }, + cancel, + completion, + }; +} diff --git a/packages/core/src/db/migrations/20260831000000_create_goal_control_plane.js b/packages/core/src/db/migrations/20260831000000_create_goal_control_plane.js new file mode 100644 index 000000000..f5f430090 --- /dev/null +++ b/packages/core/src/db/migrations/20260831000000_create_goal_control_plane.js @@ -0,0 +1,465 @@ +/** + * Durable goal control plane (issue #2006, part of epic #2003). + * + * Long-running goals require an owned, durable source of truth instead of the + * expiring Redis records and one-shot agent jobs used by tasks. This migration + * introduces ten goal-domain tables: goal identity/lifecycle and request + * idempotency, the hierarchical node + * tree with dependencies, provider sessions, an append-only per-goal event log, + * ordered corrective messages, and the auditable state/model transition and + * pause-interval history from which elapsed/active/paused time is derived. + * + * Invariants that require multi-row reasoning (monotonic sequence allocation, + * fenced lease commits, optimistic version bumps, transition validity) are + * enforced by the repository/service layer inside transactions. The schema + * enforces referential integrity, enumerations, uniqueness, and non-negativity + * so a corrupt row cannot be persisted even if application code regresses. + * + * Forward-compatible: it only adds tables and leaves existing task/planner/issue + * schema untouched. + */ + +const ISO_NOW_SQL = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + +function isoNow(knex) { + return knex.raw(`(${ISO_NOW_SQL})`); +} + +const GOAL_STATES = [ + 'queued', 'planning', 'running', 'pausing', 'paused', 'recovering', + 'completing', 'completed', 'failed', 'cancelled', +]; +const NODE_KINDS = ['root_epic', 'sub_epic', 'implementation_issue', 'implementation_pr']; +const NODE_STATUSES = ['pending', 'in_progress', 'blocked', 'completed', 'failed', 'cancelled']; +const EVENT_KINDS = ['lifecycle', 'output', 'domain']; +const MERGE_POLICIES = ['manual', 'auto', 'auto_squash']; + +export async function up(knex) { + await knex.schema.createTable('goals', (table) => { + table.text('goal_id').notNullable().primary(); + table.text('owner_user_id').notNullable(); + table.text('repository').notNullable(); + table.text('objective').notNullable(); + table.text('state').notNullable().defaultTo('queued').checkIn(GOAL_STATES); + table.text('agent').notNullable(); + table.text('requested_model').notNullable(); + table.text('effective_model').notNullable(); + table.integer('max_active_tasks').notNullable().defaultTo(3); + table.boolean('ultrafix_enabled').notNullable().defaultTo(false); + table.integer('ultrafix_goal').nullable(); + table.integer('ultrafix_max_cycles').nullable(); + table + .text('merge_policy') + .notNullable() + .defaultTo('manual') + .checkIn(MERGE_POLICIES); + table.integer('version').notNullable().defaultTo(1); + // Fenced controller lease. A holder owns the goal for the epoch it claimed; + // a takeover strictly increases the epoch so stale holders can be detected. + table.text('lease_owner').nullable(); + table.integer('lease_epoch').notNullable().defaultTo(0); + table.text('lease_expires_at').nullable(); + table.text('terminal_reason').nullable(); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + table.text('updated_at').notNullable().defaultTo(isoNow(knex)); + + table.check( + 'typeof(max_active_tasks) = \'integer\' AND max_active_tasks >= 1 AND max_active_tasks <= 20', + {}, + 'goals_max_active_tasks_check' + ); + table.check( + 'typeof(version) = \'integer\' AND version >= 1', + {}, + 'goals_version_check' + ); + table.check( + 'typeof(lease_epoch) = \'integer\' AND lease_epoch >= 0', + {}, + 'goals_lease_epoch_check' + ); + table.check( + 'ultrafix_enabled IN (0, 1)', + {}, + 'goals_ultrafix_boolean_check' + ); + table.check( + "terminal_reason IS NULL OR terminal_reason IN ('objective_met', 'user_cancelled', 'unrecoverable_error', 'concurrency_exhausted', 'superseded')", + {}, + 'goals_terminal_reason_check' + ); + table.check( + '(ultrafix_enabled = 0 AND ultrafix_goal IS NULL AND ultrafix_max_cycles IS NULL) OR (ultrafix_enabled = 1 AND typeof(ultrafix_goal) = \'integer\' AND ultrafix_goal BETWEEN 1 AND 10 AND typeof(ultrafix_max_cycles) = \'integer\' AND ultrafix_max_cycles BETWEEN 1 AND 20)', + {}, + 'goals_ultrafix_settings_check' + ); + table.check( + "length(trim(goal_id)) BETWEEN 1 AND 255 AND length(trim(owner_user_id)) BETWEEN 1 AND 255 AND length(trim(repository)) BETWEEN 1 AND 255 AND length(trim(objective)) BETWEEN 1 AND 4000 AND length(agent) BETWEEN 1 AND 255 AND length(requested_model) BETWEEN 1 AND 255 AND length(effective_model) BETWEEN 1 AND 255 AND (lease_owner IS NULL OR length(lease_owner) BETWEEN 1 AND 255)", + {}, + 'goals_required_text_check' + ); + + table.index('owner_user_id', 'goals_owner_idx'); + table.index('repository', 'goals_repository_idx'); + table.index(['owner_user_id', 'repository'], 'goals_owner_repository_idx'); + table.index(['owner_user_id', 'state'], 'goals_owner_state_idx'); + }); + + await knex.schema.createTable('goal_idempotency_keys', (table) => { + table.text('owner_user_id').notNullable(); + table.text('operation').notNullable(); + table.text('idempotency_key').notNullable(); + table.text('request_hash').notNullable(); + table.text('claim_token').nullable(); + // Nullable until the claimant commits its effect. The primary key itself is + // the atomic reservation used by cross-connection idempotent requests. + table.text('goal_id').nullable(); + table.text('response_json').nullable(); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + table.primary(['owner_user_id', 'operation', 'idempotency_key']); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + table.index(['goal_id', 'operation'], 'goal_idempotency_goal_operation_idx'); + table.check( + 'length(owner_user_id) BETWEEN 1 AND 255 AND length(operation) BETWEEN 1 AND 512 AND length(idempotency_key) BETWEEN 1 AND 255 AND (claim_token IS NULL OR length(claim_token) BETWEEN 1 AND 255)', + {}, + 'goal_idempotency_text_bounds_check' + ); + }); + + await knex.schema.createTable('goal_nodes', (table) => { + table.text('node_id').notNullable().primary(); + // Preserve whether the caller explicitly selected an identifier. The + // generated node_id alone cannot distinguish an omitted ID from a retry + // that supplies the generated response ID, which are different requests. + table.text('requested_node_id').nullable(); + table.text('goal_id').notNullable(); + table.text('parent_node_id').nullable(); + table.text('kind').notNullable().checkIn(NODE_KINDS); + // Stable per-goal idempotency key so replanning does not duplicate nodes. + table.text('idempotency_key').notNullable(); + // External GitHub identity (issue/PR number and its kind), when materialized. + table.text('external_ref').nullable(); + table.text('external_kind').nullable(); + table.text('title').nullable(); + table.text('status').notNullable().defaultTo('pending').checkIn(NODE_STATUSES); + table.integer('attempt_count').notNullable().defaultTo(0); + table.integer('order_index').notNullable().defaultTo(0); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + table.text('updated_at').notNullable().defaultTo(isoNow(knex)); + + table.check( + 'typeof(attempt_count) = \'integer\' AND attempt_count >= 0', + {}, + 'goal_nodes_attempt_count_check' + ); + table.check( + 'typeof(order_index) = \'integer\' AND order_index >= 0', + {}, + 'goal_nodes_order_index_check' + ); + table.check( + 'length(node_id) BETWEEN 1 AND 255 AND (requested_node_id IS NULL OR length(requested_node_id) BETWEEN 1 AND 255) AND length(idempotency_key) BETWEEN 1 AND 255 AND (parent_node_id IS NULL OR length(parent_node_id) BETWEEN 1 AND 255) AND (external_ref IS NULL OR length(external_ref) <= 255) AND (external_kind IS NULL OR length(external_kind) <= 255)', + {}, + 'goal_nodes_text_bounds_check' + ); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + table.unique(['goal_id', 'node_id'], { + indexName: 'goal_nodes_goal_node_idx', + }); + table + .foreign(['goal_id', 'parent_node_id']) + .references(['goal_id', 'node_id']) + .inTable('goal_nodes') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.unique(['goal_id', 'idempotency_key'], { + indexName: 'goal_nodes_goal_idempotency_idx', + }); + table.index(['goal_id', 'parent_node_id', 'order_index'], 'goal_nodes_tree_idx'); + table.index(['goal_id', 'status'], 'goal_nodes_status_idx'); + }); + + await knex.schema.createTable('goal_node_dependencies', (table) => { + table.text('goal_id').notNullable(); + table.text('node_id').notNullable(); + table.text('depends_on_node_id').notNullable(); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + + table.primary(['goal_id', 'node_id', 'depends_on_node_id']); + table.check( + 'node_id <> depends_on_node_id', + {}, + 'goal_node_dependencies_no_self_edge_check' + ); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + table + .foreign(['goal_id', 'node_id']) + .references(['goal_id', 'node_id']) + .inTable('goal_nodes') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + table + .foreign(['goal_id', 'depends_on_node_id']) + .references(['goal_id', 'node_id']) + .inTable('goal_nodes') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.index('goal_id', 'goal_node_dependencies_goal_idx'); + table.index('depends_on_node_id', 'goal_node_dependencies_dependency_idx'); + }); + + await knex.schema.createTable('goal_provider_sessions', (table) => { + table.text('session_id').notNullable().primary(); + table.text('goal_id').notNullable(); + table.text('agent').notNullable(); + table.text('provider_thread_id').nullable(); + table.text('runtime_id').nullable(); + table.text('worktree_id').nullable(); + table.text('last_checkpoint').nullable(); + table.text('effective_model').notNullable(); + table.text('recovery_metadata_json').nullable(); + // Fenced lease generation this session belongs to; a stale generation must + // not resume authoritative provider work after a controller takeover. + table.integer('lease_generation').notNullable().defaultTo(0); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + table.text('updated_at').notNullable().defaultTo(isoNow(knex)); + + table.check( + 'typeof(lease_generation) = \'integer\' AND lease_generation >= 0', + {}, + 'goal_provider_sessions_lease_generation_check' + ); + table.check( + 'length(session_id) BETWEEN 1 AND 255 AND length(agent) BETWEEN 1 AND 255 AND (provider_thread_id IS NULL OR length(provider_thread_id) <= 255) AND (runtime_id IS NULL OR length(runtime_id) <= 255) AND (worktree_id IS NULL OR length(worktree_id) <= 255)', + {}, + 'goal_provider_sessions_text_bounds_check' + ); + table.check( + 'recovery_metadata_json IS NULL OR (json_valid(recovery_metadata_json) AND length(recovery_metadata_json) <= 4096)', + {}, + 'goal_provider_sessions_recovery_metadata_check' + ); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.unique(['goal_id', 'agent'], { + indexName: 'goal_provider_sessions_goal_agent_idx', + }); + }); + + await knex.schema.createTable('goal_events', (table) => { + table.increments('id').primary(); + table.text('goal_id').notNullable(); + // Monotonic per-goal sequence; unique(goal_id, sequence) rejects gaps/dupes. + table.integer('sequence').notNullable(); + table.text('kind').notNullable().checkIn(EVENT_KINDS); + table.text('event_type').notNullable(); + table.text('payload_json').nullable(); + table.text('idempotency_key').notNullable(); + // Epoch of the controller lease that appended this event, for audit. + table.integer('lease_epoch').notNullable().defaultTo(0); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + + table.check( + 'typeof(sequence) = \'integer\' AND sequence >= 1', + {}, + 'goal_events_sequence_check' + ); + table.check( + 'length(event_type) BETWEEN 1 AND 255 AND length(idempotency_key) BETWEEN 1 AND 255', + {}, + 'goal_events_text_bounds_check' + ); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.unique(['goal_id', 'sequence'], { + indexName: 'goal_events_goal_sequence_idx', + }); + table.unique(['goal_id', 'idempotency_key'], { + indexName: 'goal_events_goal_idempotency_idx', + }); + table.index(['goal_id', 'kind', 'sequence'], 'goal_events_goal_kind_idx'); + }); + + await knex.schema.createTable('goal_messages', (table) => { + table.text('message_id').notNullable().primary(); + table.text('goal_id').notNullable(); + // Ordered delivery position within the goal. + table.integer('sequence').notNullable(); + table.text('body').notNullable(); + table.text('predefined_kind').nullable(); + // Kept as validated text rather than a database enum so the events follow-up + // can add delivery states without rebuilding this table. + table.text('state').notNullable().defaultTo('queued'); + table.text('delivered_at').nullable(); + table.text('acknowledged_at').nullable(); + table.integer('delivery_attempts').notNullable().defaultTo(0); + table.text('last_error').nullable(); + table.text('idempotency_key').notNullable(); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + + table.check( + 'typeof(sequence) = \'integer\' AND sequence >= 1', + {}, + 'goal_messages_sequence_check' + ); + table.check( + 'length(trim(body)) BETWEEN 1 AND 4000', + {}, + 'goal_messages_body_check' + ); + table.check( + "state IN ('queued', 'delivered', 'acknowledged') AND length(message_id) BETWEEN 1 AND 255 AND length(idempotency_key) BETWEEN 1 AND 255 AND (predefined_kind IS NULL OR length(predefined_kind) <= 255) AND (acknowledged_at IS NULL OR delivered_at IS NOT NULL)", + {}, + 'goal_messages_state_consistency_check' + ); + table.check( + "typeof(delivery_attempts) = 'integer' AND delivery_attempts >= 0", + {}, + 'goal_messages_delivery_attempts_check' + ); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.unique(['goal_id', 'sequence'], { + indexName: 'goal_messages_goal_sequence_idx', + }); + table.unique(['goal_id', 'idempotency_key'], { + indexName: 'goal_messages_goal_idempotency_idx', + }); + table.index(['goal_id', 'state', 'sequence'], 'goal_messages_delivery_idx'); + }); + + await knex.schema.createTable('goal_state_transitions', (table) => { + table.increments('id').primary(); + table.text('goal_id').notNullable(); + table.text('from_state').notNullable().checkIn(GOAL_STATES); + table.text('to_state').notNullable().checkIn(GOAL_STATES); + table.text('reason').nullable(); + table.integer('lease_epoch').notNullable().defaultTo(0); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.index(['goal_id', 'created_at'], 'goal_state_transitions_goal_idx'); + table.check( + 'reason IS NULL OR length(reason) <= 1000', + {}, + 'goal_state_transitions_reason_bounds_check' + ); + }); + + await knex.schema.createTable('goal_model_transitions', (table) => { + table.increments('id').primary(); + table.text('goal_id').notNullable(); + table.text('previous_model').notNullable(); + table.text('requested_model').notNullable(); + table.text('effective_model').notNullable(); + // Requested changes are recorded unapplied until a runtime acknowledges the + // change at a safe boundary, at which point the effective model advances. + table.boolean('applied').notNullable().defaultTo(false); + table.text('reason').nullable(); + table.text('created_at').notNullable().defaultTo(isoNow(knex)); + table.text('applied_at').nullable(); + + table.check('applied IN (0, 1)', {}, 'goal_model_transitions_applied_check'); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.index(['goal_id', 'created_at'], 'goal_model_transitions_goal_idx'); + table.check( + 'length(previous_model) BETWEEN 1 AND 255 AND length(requested_model) BETWEEN 1 AND 255 AND length(effective_model) BETWEEN 1 AND 255 AND (reason IS NULL OR length(reason) <= 1000)', + {}, + 'goal_model_transitions_text_bounds_check' + ); + }); + + await knex.schema.createTable('goal_pause_intervals', (table) => { + table.increments('id').primary(); + table.text('goal_id').notNullable(); + table.text('paused_at').notNullable(); + table.text('resumed_at').nullable(); + table.text('reason').nullable(); + + table.check( + 'resumed_at IS NULL OR resumed_at >= paused_at', + {}, + 'goal_pause_intervals_order_check' + ); + + table + .foreign('goal_id') + .references('goal_id') + .inTable('goals') + .onUpdate('RESTRICT') + .onDelete('CASCADE'); + + table.index(['goal_id', 'paused_at'], 'goal_pause_intervals_goal_idx'); + table.check( + 'reason IS NULL OR length(reason) <= 1000', + {}, + 'goal_pause_intervals_reason_bounds_check' + ); + }); + + // At most one open pause interval per goal so active-time accounting stays + // unambiguous. A closed interval has resumed_at set and is not indexed here. + await knex.raw(` + CREATE UNIQUE INDEX goal_pause_intervals_open_idx + ON goal_pause_intervals (goal_id) + WHERE resumed_at IS NULL + `); +} + +export async function down(knex) { + await knex.schema.dropTableIfExists('goal_pause_intervals'); + await knex.schema.dropTableIfExists('goal_model_transitions'); + await knex.schema.dropTableIfExists('goal_state_transitions'); + await knex.schema.dropTableIfExists('goal_messages'); + await knex.schema.dropTableIfExists('goal_events'); + await knex.schema.dropTableIfExists('goal_provider_sessions'); + await knex.schema.dropTableIfExists('goal_node_dependencies'); + await knex.schema.dropTableIfExists('goal_nodes'); + await knex.schema.dropTableIfExists('goal_idempotency_keys'); + await knex.schema.dropTableIfExists('goals'); +} diff --git a/packages/core/src/db/migrations/20260901000000_add_durable_goal_replay.js b/packages/core/src/db/migrations/20260901000000_add_durable_goal_replay.js new file mode 100644 index 000000000..91998dfbf --- /dev/null +++ b/packages/core/src/db/migrations/20260901000000_add_durable_goal_replay.js @@ -0,0 +1,233 @@ +/** + * Durable goal event/replay/message projections (issue #2008). + * + * This migration deliberately owns only append/audit and read projections. It + * does not mutate controller plan/node/attempt authority owned by the goal + * reconciler. All derived rows can be rebuilt from the retained event log and + * its compaction checkpoints. + */ + +const ISO_NOW_SQL = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + +export async function up(knex) { + // A migration source may stage this leaf before the foundation branch is + // present (serialized merge-order validation). In that ordering this leaf is + // an intentional no-op; a fresh/full chain orders the timestamped foundation + // first and installs the durable tables below. + if (!await knex.schema.hasTable('goal_events')) return; + for (const [column, add] of [ + ['schema_version', table => table.integer('schema_version').notNullable().defaultTo(1)], + ['source_session_id', table => table.text('source_session_id').nullable()], + ['source_turn_id', table => table.text('source_turn_id').nullable()], + ['source_execution_id', table => table.text('source_execution_id').nullable()], + ['source_attempt_id', table => table.text('source_attempt_id').nullable()], + ['source_provider_sequence', table => table.integer('source_provider_sequence').nullable()], + ['source_chunk_index', table => table.integer('source_chunk_index').nullable()], + ['lease_generation', table => table.integer('lease_generation').nullable()], + ['payload_bytes', table => table.integer('payload_bytes').notNullable().defaultTo(0)], + ]) { + if (!await knex.schema.hasColumn('goal_events', column)) { + await knex.schema.alterTable('goal_events', add); + } + } + + await knex.raw(` + CREATE UNIQUE INDEX IF NOT EXISTS goal_events_source_occurrence_idx + ON goal_events ( + goal_id, source_session_id, source_turn_id, source_execution_id, + source_attempt_id, source_provider_sequence, source_chunk_index, + lease_generation + ) + WHERE source_session_id IS NOT NULL + `); + await knex('goal_events').where('payload_bytes', 0).whereNotNull('payload_json') + .update({ payload_bytes: knex.raw('length(CAST(payload_json AS BLOB))') }); + + for (const [column, add] of [ + ['current_turn_id', table => table.text('current_turn_id').nullable()], + ['current_execution_id', table => table.text('current_execution_id').nullable()], + ['current_attempt_id', table => table.text('current_attempt_id').nullable()], + ]) { + if (!await knex.schema.hasColumn('goal_provider_sessions', column)) { + await knex.schema.alterTable('goal_provider_sessions', add); + } + } + + await rebuildMessages(knex); + + await knex.schema.createTable('goal_event_state', table => { + table.text('goal_id').primary().notNullable(); + table.integer('high_watermark').notNullable().defaultTo(0); + table.integer('min_retained_sequence').notNullable().defaultTo(1); + table.integer('projection_sequence').notNullable().defaultTo(0); + table.integer('checkpoint_sequence').notNullable().defaultTo(0); + table.text('updated_at').notNullable().defaultTo(knex.raw(`(${ISO_NOW_SQL})`)); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + await knex.raw(` + INSERT INTO goal_event_state ( + goal_id, high_watermark, min_retained_sequence, projection_sequence, + checkpoint_sequence, updated_at + ) + SELECT g.goal_id, COALESCE(MAX(e.sequence), 0), 1, COALESCE(MAX(e.sequence), 0), 0, + strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + FROM goals g LEFT JOIN goal_events e ON e.goal_id = g.goal_id + GROUP BY g.goal_id + `); + + await knex.schema.createTable('goal_event_quarantine', table => { + table.increments('id').primary(); + table.text('goal_id').notNullable(); + table.text('idempotency_key').notNullable(); + table.text('event_type').nullable(); + table.text('reason').notNullable(); + table.text('payload_digest').notNullable(); + table.text('created_at').notNullable().defaultTo(knex.raw(`(${ISO_NOW_SQL})`)); + table.unique(['goal_id', 'idempotency_key']); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + + await knex.schema.createTable('goal_usage_occurrences', table => { + table.increments('id').primary(); + table.text('goal_id').notNullable(); + table.text('provider').notNullable(); + table.text('model').notNullable(); + table.text('session_id').notNullable(); + table.text('execution_id').notNullable(); + table.text('attempt_id').notNullable(); + table.text('occurrence_id').notNullable(); + table.integer('input_tokens').notNullable().defaultTo(0); + table.integer('output_tokens').notNullable().defaultTo(0); + table.integer('cache_read_tokens').notNullable().defaultTo(0); + table.integer('cache_write_tokens').notNullable().defaultTo(0); + table.integer('reasoning_tokens').notNullable().defaultTo(0); + table.integer('event_sequence').notNullable(); + table.text('created_at').notNullable().defaultTo(knex.raw(`(${ISO_NOW_SQL})`)); + table.unique( + ['goal_id', 'provider', 'model', 'session_id', 'execution_id', 'attempt_id', 'occurrence_id'], + { indexName: 'goal_usage_occurrence_identity_idx' } + ); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + + await knex.schema.createTable('goal_usage_watermarks', table => { + table.text('goal_id').notNullable(); + table.text('provider').notNullable(); + table.text('model').notNullable(); + table.text('session_id').notNullable(); + table.text('execution_id').notNullable(); + table.text('attempt_id').notNullable(); + table.integer('input_tokens').notNullable().defaultTo(0); + table.integer('output_tokens').notNullable().defaultTo(0); + table.integer('cache_read_tokens').notNullable().defaultTo(0); + table.integer('cache_write_tokens').notNullable().defaultTo(0); + table.integer('reasoning_tokens').notNullable().defaultTo(0); + table.primary(['goal_id', 'provider', 'model', 'session_id', 'execution_id', 'attempt_id']); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + + await knex.schema.createTable('goal_external_projections', table => { + table.text('goal_id').notNullable(); + table.text('entity_type').notNullable(); + table.integer('entity_number').notNullable(); + table.text('status').notNullable(); + table.integer('event_sequence').notNullable(); + table.text('updated_at').notNullable(); + table.primary(['goal_id', 'entity_type', 'entity_number']); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + + await knex.schema.createTable('goal_provider_todos', table => { + table.text('goal_id').notNullable(); + table.text('session_id').notNullable(); + table.text('todo_id').notNullable(); + table.text('body').notNullable(); + table.text('status').notNullable(); + table.integer('event_sequence').notNullable(); + table.primary(['goal_id', 'session_id', 'todo_id']); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + + await knex.schema.createTable('goal_compaction_checkpoints', table => { + table.text('goal_id').notNullable(); + table.integer('through_sequence').notNullable(); + table.text('content_digest').notNullable(); + table.integer('removed_event_count').notNullable(); + table.integer('removed_payload_bytes').notNullable(); + table.text('created_at').notNullable().defaultTo(knex.raw(`(${ISO_NOW_SQL})`)); + table.primary(['goal_id', 'through_sequence']); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + + if (!await knex.schema.hasColumn('goal_model_transitions', 'outcome')) { + await knex.schema.alterTable('goal_model_transitions', table => { + table.text('outcome').notNullable().defaultTo('pending'); + table.text('superseded_at').nullable(); + }); + await knex('goal_model_transitions').where('applied', 1).update({ outcome: 'applied' }); + } +} + +async function rebuildMessages(knex) { + if (await knex.schema.hasColumn('goal_messages', 'queue_ordinal')) return; + await knex.raw('DROP INDEX IF EXISTS goal_messages_goal_sequence_idx'); + await knex.raw('DROP INDEX IF EXISTS goal_messages_goal_idempotency_idx'); + await knex.raw('DROP INDEX IF EXISTS goal_messages_delivery_idx'); + await knex.schema.renameTable('goal_messages', 'goal_messages_foundation'); + await knex.schema.createTable('goal_messages', table => { + table.text('message_id').primary().notNullable(); + table.text('goal_id').notNullable(); + table.integer('sequence').notNullable(); + table.integer('queue_ordinal').notNullable(); + table.text('body').notNullable(); + table.text('predefined_kind').nullable(); + table.text('canned_action').nullable(); + table.text('author_user_id').nullable(); + table.text('state').notNullable().defaultTo('queued'); + table.text('claimed_by').nullable(); + table.text('claimed_turn_id').nullable(); + table.integer('claimed_lease_generation').nullable(); + table.text('delivery_key').nullable(); + table.text('delivered_at').nullable(); + table.text('acknowledged_at').nullable(); + table.text('cancelled_at').nullable(); + table.text('failed_at').nullable(); + table.integer('delivery_attempts').notNullable().defaultTo(0); + table.integer('retry_count').notNullable().defaultTo(0); + table.text('last_error').nullable(); + table.text('idempotency_key').notNullable(); + table.integer('enqueue_event_sequence').nullable(); + table.integer('state_event_sequence').nullable(); + table.text('created_at').notNullable(); + table.check("state IN ('queued','delivering','delivered','acknowledged','failed','cancelled')"); + table.check("canned_action IS NULL OR canned_action IN ('whats_done','whats_left')"); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + table.unique(['goal_id', 'sequence'], { indexName: 'goal_messages_goal_sequence_idx' }); + table.unique(['goal_id', 'queue_ordinal'], { indexName: 'goal_messages_goal_ordinal_idx' }); + table.unique(['goal_id', 'idempotency_key'], { indexName: 'goal_messages_goal_idempotency_idx' }); + table.index(['goal_id', 'state', 'queue_ordinal'], 'goal_messages_delivery_idx'); + }); + await knex.raw(` + INSERT INTO goal_messages ( + message_id, goal_id, sequence, queue_ordinal, body, predefined_kind, + state, delivered_at, acknowledged_at, delivery_attempts, retry_count, + last_error, idempotency_key, created_at + ) + SELECT message_id, goal_id, sequence, sequence, body, predefined_kind, + state, delivered_at, acknowledged_at, delivery_attempts, 0, + last_error, idempotency_key, created_at + FROM goal_messages_foundation + `); + await knex.schema.dropTable('goal_messages_foundation'); +} + +export async function down(knex) { + await knex.schema.dropTableIfExists('goal_compaction_checkpoints'); + await knex.schema.dropTableIfExists('goal_provider_todos'); + await knex.schema.dropTableIfExists('goal_external_projections'); + await knex.schema.dropTableIfExists('goal_usage_watermarks'); + await knex.schema.dropTableIfExists('goal_usage_occurrences'); + await knex.schema.dropTableIfExists('goal_event_quarantine'); + await knex.schema.dropTableIfExists('goal_event_state'); + await knex.raw('DROP INDEX IF EXISTS goal_events_source_occurrence_idx'); +} diff --git a/packages/core/src/db/migrations/20260902000000_extend_goal_control_provider_effects.js b/packages/core/src/db/migrations/20260902000000_extend_goal_control_provider_effects.js new file mode 100644 index 000000000..d5daf6739 --- /dev/null +++ b/packages/core/src/db/migrations/20260902000000_extend_goal_control_provider_effects.js @@ -0,0 +1,79 @@ +/** + * Goal-session runtime extension for the control database. goal_events and + * goal_messages are deliberately not created here: they belong to the #2018 + * control-plane migration and are consumed by the runtime adapter. + */ +export async function up(knex) { + await knex.schema.createTable('goal_session_runtime_state', (table) => { + table.string('session_id').primary(); + table.string('goal_id').notNullable(); + table.string('scope').notNullable().unique(); + table.text('payload_json').notNullable(); + table.foreign('session_id').references('session_id').inTable('goal_provider_sessions').onDelete('CASCADE'); + table.foreign('goal_id').references('goal_id').inTable('goals').onDelete('CASCADE'); + }); + await knex.schema.createTable('goal_session_runtime_commits', (table) => { + table.string('session_id').notNullable(); + table.string('goal_id').notNullable(); + table.string('kind').notNullable(); + table.string('identity').notNullable(); + table.primary(['kind', 'identity']); + table.foreign('session_id').references('session_id').inTable('goal_provider_sessions').onDelete('CASCADE'); + }); + await knex.schema.createTable('goal_session_runtime_model_changes', (table) => { + table.string('session_id').notNullable(); + table.string('goal_id').notNullable(); + table.string('scope').notNullable(); + table.string('operation_id').notNullable(); + table.integer('sequence').notNullable(); + table.string('model').notNullable(); + table.string('status').notNullable(); + table.text('acknowledgement_json'); + table.primary(['scope', 'operation_id']); + table.unique(['scope', 'sequence']); + table.foreign('session_id').references('session_id').inTable('goal_provider_sessions').onDelete('CASCADE'); + }); + await knex.schema.createTable('goal_session_runtime_model_sequences', (table) => { + table.string('session_id').notNullable(); + table.string('goal_id').notNullable(); + table.string('scope').primary(); + table.integer('next_sequence').notNullable(); + table.foreign('session_id').references('session_id').inTable('goal_provider_sessions').onDelete('CASCADE'); + }); + await knex.schema.createTable('goal_session_runtime_provider_effects', (table) => { + table.string('session_id').notNullable(); + table.string('goal_id').notNullable(); + table.string('scope').notNullable(); + table.string('operation_id').notNullable(); + table.string('kind').notNullable(); + table.string('stage').notNullable(); + table.string('status').notNullable(); + table.string('claim_token').notNullable(); + table.text('outcome_json'); + table.timestamp('updated_at').notNullable(); + table.primary(['scope', 'operation_id', 'stage']); + table.foreign('session_id').references('session_id').inTable('goal_provider_sessions').onDelete('CASCADE'); + table.check("kind IN ('open','turn','resume','reconcile','steer','model','pause','cancel')"); + table.check("stage IN ('provider_primitive','stream_first_next','container_spawn')"); + table.check("status IN ('unstarted','started','recoverable','settled','poisoned')"); + table.check('length(operation_id) BETWEEN 1 AND 255 AND length(claim_token) BETWEEN 1 AND 255'); + }); + await knex.raw(`CREATE TRIGGER goal_runtime_provider_effect_stage_insert + BEFORE INSERT ON goal_session_runtime_provider_effects + WHEN NEW.stage NOT IN ('provider_primitive', 'stream_first_next', 'container_spawn') + BEGIN SELECT RAISE(ABORT, 'invalid provider effect stage'); END`); + await knex.raw(`CREATE TRIGGER goal_runtime_provider_effect_stage_update + BEFORE UPDATE OF stage ON goal_session_runtime_provider_effects + WHEN NEW.stage NOT IN ('provider_primitive', 'stream_first_next', 'container_spawn') + BEGIN SELECT RAISE(ABORT, 'invalid provider effect stage'); END`); +} + +export async function down(knex) { + await knex.raw('DROP TRIGGER IF EXISTS goal_runtime_provider_effect_stage_update'); + await knex.raw('DROP TRIGGER IF EXISTS goal_runtime_provider_effect_stage_insert'); + await knex.schema.dropTableIfExists('goal_session_runtime_provider_effects'); + await knex.schema.dropTableIfExists('goal_session_runtime_model_sequences'); + await knex.schema.dropTableIfExists('goal_session_runtime_model_changes'); + await knex.schema.dropTableIfExists('goal_session_runtime_commits'); + await knex.schema.dropTableIfExists('goal_session_runtime_state'); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9efd78e4b..85e098f5b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -233,8 +233,8 @@ export { buildClaudePrompt } from './claude/claudeHelpers.js'; export type { ClaudeOutput, ConversationLogEntry, ClaudeOutputResult, BuildClaudePromptOptions, DockerArgsParams, StorePromptOptions } from './claude/claudeHelpers.js'; -export { buildPlannerAbortSignalKey, executeDockerCommand, findRunningDockerContainerForTask, findTaskContainer, inspectLegacyDockerContainerLivenessForTask, runWithExecutionAbortSignal, runWithPlannerAbortContext, stopDockerContainer, ExecutionAbortedError, ensureAgentBundleImage } from './claude/docker/dockerExecutor.js'; -export type { RunningTaskContainer } from './claude/docker/dockerExecutor.js'; +export { buildPlannerAbortSignalKey, executeDockerCommand, executeSupervisedDockerCommand, findRunningDockerContainerForTask, findTaskContainer, inspectLegacyDockerContainerLivenessForTask, runWithExecutionAbortSignal, runWithPlannerAbortContext, stopDockerContainer, ExecutionAbortedError, ensureAgentBundleImage } from './claude/docker/dockerExecutor.js'; +export type { RunningTaskContainer, SupervisedDockerExecution, SupervisedDockerFence, SupervisedDockerOptions, SupervisedDockerOutput } from './claude/docker/dockerExecutor.js'; export { cleanupUnusedAgentImages, listAgentImages } from './claude/docker/dockerImageManager.js'; export type { VersionedImageBuildResult } from './claude/docker/dockerExecutor.js'; export { @@ -348,7 +348,7 @@ export { CONTAINER_CONFIG_PATHS } from './agents/types.js'; export { DEFAULT_CONFIG_PATHS, resolveConfigPath, getDefaultConfigPath, loadAgents, loadEffectiveAgentBaseImages, migrateAgentConfigs } from './config/configManager.js'; // Agent version management -export * from './agents/version/index.js'; +export * from './agents/index.js'; // Repository chat message persistence export { diff --git a/packages/core/src/types/better-sqlite3.d.ts b/packages/core/src/types/better-sqlite3.d.ts new file mode 100644 index 000000000..b247cbde1 --- /dev/null +++ b/packages/core/src/types/better-sqlite3.d.ts @@ -0,0 +1,24 @@ +declare module 'better-sqlite3' { + interface RunResult { changes: number } + interface Statement { + run(...parameters: unknown[]): RunResult; + get(...parameters: unknown[]): unknown; + all(...parameters: unknown[]): unknown[]; + } + interface Transaction { + (): T; + immediate(): T; + } + class BetterSqlite3Database { + constructor(filename: string); + pragma(source: string): unknown; + exec(source: string): void; + prepare(source: string): Statement; + transaction(operation: () => T): Transaction; + close(): void; + } + namespace BetterSqlite3Database { + type Database = BetterSqlite3Database; + } + export default BetterSqlite3Database; +} diff --git a/packages/core/test/SqliteGoalSessionTestPorts.ts b/packages/core/test/SqliteGoalSessionTestPorts.ts new file mode 100644 index 000000000..c92b0ac54 --- /dev/null +++ b/packages/core/test/SqliteGoalSessionTestPorts.ts @@ -0,0 +1,551 @@ +import Database from 'better-sqlite3'; +import type { + DurableCorrectiveMessage, + GoalContainerInspection, + GoalEventAppendResult, + GoalExecutionIdentity, + GoalRepositoryIdentity, + GoalRepositoryInspection, + GoalSessionControlFence, + GoalSessionControlTransition, + GoalSessionEvent, + GoalSessionFence, + GoalSessionIdentity, + GoalModelChangeAcknowledgement, + GoalModelChangeHistoryRecord, + GoalProviderEffectStage, + GoalProviderOperationFence, + GoalStartedProviderEffect, + GoalSessionRuntimePorts, + GoalSessionState, + GoalTerminalCommit, + PersistedGoalSessionEvent, +} from '../src/agents/goalSession/contract.js'; +import { sanitizeGoalSessionEvent } from '../src/agents/goalSession/securityBoundary.js'; +import { assertProviderFirstEffectState } from '../src/agents/goalSession/providerFirstEffect.js'; +import { assertStartedProviderEffect } from '../src/agents/goalSession/providerEffectProtocol.js'; +import { + AuthoritativeGoalSessionRuntimePorts, +} from '../src/agents/goalSession/AuthoritativeGoalSessionRuntimePorts.js'; +import { GoalSessionContractError } from '../src/agents/goalSession/errors.js'; +import { GoalSessionScopeError } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; + +function scope(identity: GoalSessionIdentity): string { + return `${identity.goalId}\0${identity.sessionId}`; +} + +function clone(value: T): T { return structuredClone(value); } + +/** Separate SQLite connections over one file, used only for true cross-port durability tests. */ +export class SqliteGoalSessionTestPorts { + private readonly database: Database.Database; + private transitionFault: 'before_commit' | 'after_commit' | undefined; + private providerFault: 'receipt_write' | 'commit' | undefined; + + constructor(filename: string) { + this.database = new Database(filename); + this.database.pragma('journal_mode = WAL'); + this.database.pragma('busy_timeout = 5000'); + this.database.exec(` + CREATE TABLE IF NOT EXISTS goal_session_runtime_owners ( + session_id TEXT PRIMARY KEY, goal_id TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS goal_session_runtime_state (scope TEXT PRIMARY KEY, payload TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS goal_session_runtime_events ( + scope TEXT NOT NULL, sequence INTEGER NOT NULL, payload TEXT NOT NULL, + PRIMARY KEY (scope, sequence) + ); + CREATE TABLE IF NOT EXISTS goal_session_runtime_commits (kind TEXT NOT NULL, identity TEXT NOT NULL, PRIMARY KEY (kind, identity)); + CREATE TABLE IF NOT EXISTS goal_session_runtime_messages ( + scope TEXT NOT NULL, message_id TEXT NOT NULL, sequence INTEGER NOT NULL, payload TEXT NOT NULL, + PRIMARY KEY (scope, message_id) + ); + CREATE TABLE IF NOT EXISTS goal_session_runtime_fixtures (kind TEXT NOT NULL, identity TEXT NOT NULL, payload TEXT NOT NULL, + PRIMARY KEY (kind, identity)); + CREATE TABLE IF NOT EXISTS goal_session_runtime_model_changes ( + scope TEXT NOT NULL, operation_id TEXT NOT NULL, sequence INTEGER NOT NULL, + model TEXT NOT NULL, status TEXT NOT NULL, acknowledgement TEXT, + PRIMARY KEY (scope, operation_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS goal_model_change_order + ON goal_session_runtime_model_changes(scope, sequence); + CREATE TABLE IF NOT EXISTS goal_session_runtime_model_sequences ( + scope TEXT PRIMARY KEY, next_sequence INTEGER NOT NULL CHECK(next_sequence > 0) + ); + CREATE TABLE IF NOT EXISTS goal_session_runtime_provider_effects ( + scope TEXT NOT NULL, operation_id TEXT NOT NULL, kind TEXT NOT NULL, + stage TEXT NOT NULL, status TEXT NOT NULL, outcome_json TEXT, + PRIMARY KEY (scope, operation_id, stage) + ); + INSERT OR IGNORE INTO goal_session_runtime_model_sequences(scope, next_sequence) + SELECT scope, sequence + 1 FROM ( + SELECT scope, sequence, + ROW_NUMBER() OVER (PARTITION BY scope ORDER BY sequence DESC) AS ordering_rank + FROM goal_session_runtime_model_changes + ) WHERE ordering_rank = 1; + `); + } + + asRuntimePorts(): GoalSessionRuntimePorts { + return new AuthoritativeGoalSessionRuntimePorts({ + state: this, transitions: this, events: this, terminal: this, + messages: this, modelChanges: this, providerEffects: this, + }, this).asRuntimePorts(); + } + + async claim( + identity: GoalSessionIdentity, + operationId: string, + model: string, + ): Promise { + this.assertGoalScope(identity); + return this.database.transaction(() => { + const existing = this.readModelChange(identity, operationId); + if (existing) return existing; + const row = this.database.prepare(` + INSERT INTO goal_session_runtime_model_sequences(scope, next_sequence) VALUES (?, 2) + ON CONFLICT(scope) DO UPDATE SET next_sequence = next_sequence + 1 + RETURNING next_sequence - 1 AS sequence + `).get(scope(identity)) as { sequence: number }; + this.database.prepare( + 'INSERT INTO goal_session_runtime_model_changes(scope, operation_id, sequence, model, status) VALUES (?, ?, ?, ?, ?)', + ).run(scope(identity), operationId, row.sequence, model, 'pending'); + return { operationId, model, sequence: row.sequence, status: 'pending' as const }; + }).immediate(); + } + + async settle( + identity: GoalSessionIdentity, + operationId: string, + acknowledgement: GoalModelChangeAcknowledgement, + ): Promise { + this.assertGoalScope(identity); + this.database.transaction(() => { + this.database.prepare( + 'UPDATE goal_session_runtime_model_changes SET status = ?, acknowledgement = ? WHERE scope = ? AND operation_id = ?', + ).run('settled', JSON.stringify(acknowledgement), scope(identity), operationId); + this.database.prepare(` + UPDATE goal_session_runtime_model_changes SET status = 'retired', acknowledgement = NULL + WHERE scope = ? AND status = 'settled' AND operation_id NOT IN ( + SELECT operation_id FROM goal_session_runtime_model_changes + WHERE scope = ? AND status = 'settled' ORDER BY sequence DESC LIMIT 64 + ) + `).run(scope(identity), scope(identity)); + }).immediate(); + } + + close(): void { this.database.close(); } + + async claimProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + ): Promise { + return this.database.transaction(() => { + this.assertGoalScope(fence); + assertProviderFirstEffectState(this.readState(fence), fence); + const result = this.database.prepare( + `INSERT OR IGNORE INTO goal_session_runtime_provider_effects + (scope, operation_id, kind, stage, status) VALUES (?, ?, ?, ?, 'claimed')`, + ).run(scope(fence), fence.operationId, fence.kind, stage); + if (result.changes === 1) return { status: 'claimed' as const, token: 'test-effect-token' }; + const existing = this.database.prepare(`SELECT status, outcome_json + FROM goal_session_runtime_provider_effects WHERE scope = ? AND operation_id = ? AND stage = ?`) + .get(scope(fence), fence.operationId, stage) as { status: string; outcome_json: string | null }; + return existing.status === 'settled' + ? { status: 'settled' as const, outcome: JSON.parse(existing.outcome_json ?? 'null') } + : { status: 'terminal_in_doubt' as const }; + }).immediate(); + } + + async settleProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + _token: string, + outcome: unknown, + ): Promise { + this.database.prepare(`UPDATE goal_session_runtime_provider_effects SET status = 'settled', outcome_json = ? + WHERE scope = ? AND operation_id = ? AND stage = ?`) + .run(JSON.stringify(outcome ?? null), scope(fence), fence.operationId, stage); + } + + async poisonProviderEffect(): Promise {} + + async runClaimedProviderEffect( + fence: GoalProviderOperationFence, + stage: GoalProviderEffectStage, + _token: string, + effect: () => GoalStartedProviderEffect, + ): Promise> { + let started: GoalStartedProviderEffect | undefined; + this.database.transaction(() => { + this.assertGoalScope(fence); + assertProviderFirstEffectState(this.readState(fence), fence); + const claim = this.database.prepare( + `SELECT kind, status FROM goal_session_runtime_provider_effects + WHERE scope = ? AND operation_id = ? AND stage = ?`, + ).get(scope(fence), fence.operationId, stage) as { kind: string; status: string } | undefined; + if (!claim || claim.kind !== fence.kind || claim.status !== 'claimed') throw new GoalSessionContractError( + 'Provider effect stage does not own an exact durable claim', 'PROVIDER_EFFECT_IN_DOUBT', + ); + started = effect(); + assertStartedProviderEffect(started); + if (this.providerFault === 'receipt_write') { + this.providerFault = undefined; + throw new Error('Injected provider receipt-write failure'); + } + this.database.prepare( + `UPDATE goal_session_runtime_provider_effects SET status = 'started' + WHERE scope = ? AND operation_id = ? AND stage = ? AND status = 'claimed'`, + ).run(scope(fence), fence.operationId, stage); + }).immediate(); + if (this.providerFault === 'commit') { + this.providerFault = undefined; + throw new Error('Injected provider COMMIT failure'); + } + return started!; + } + + providerEffectCount(): number { + return (this.database.prepare('SELECT COUNT(*) AS count FROM goal_session_runtime_provider_effects').get() as { count: number }).count; + } + + setTransitionFault(fault: 'before_commit' | 'after_commit' | undefined): void { + this.transitionFault = fault; + } + + setProviderFault(fault: 'receipt_write' | 'commit' | undefined): void { + this.providerFault = fault; + } + + async load(identity: GoalSessionIdentity): Promise { + this.assertGoalScope(identity); + return this.readState(identity); + } + + async create(state: Omit): Promise { + const saved = { ...clone(state), version: 1 }; + return this.database.transaction(() => { + this.assertGoalScope(state); + this.database.prepare( + 'INSERT OR IGNORE INTO goal_session_runtime_owners(session_id, goal_id) VALUES (?, ?)', + ).run(state.sessionId, state.goalId); + this.assertGoalScope(state); + const result = this.database.prepare( + 'INSERT OR IGNORE INTO goal_session_runtime_state(scope, payload) VALUES (?, ?)', + ).run(scope(state), JSON.stringify(saved)); + return result.changes === 1 ? saved : null; + }).immediate(); + } + + async compareAndSet( + expected: GoalSessionState, + next: Omit, + ): Promise { + const saved = { ...clone(next), version: expected.version + 1 }; + const current = this.readState(expected); + if (current?.version !== expected.version) return null; + const result = this.database.prepare('UPDATE goal_session_runtime_state SET payload = ? WHERE scope = ? AND payload = ?') + .run(JSON.stringify(saved), scope(expected), JSON.stringify(current)); + return result.changes === 1 ? saved : null; + } + + async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ): Promise { + const isTransition = !('scope' in operation); + if (isTransition && this.transitionFault === 'before_commit') { + this.transitionFault = undefined; + throw new Error('Injected crash before state/audit transaction commit'); + } + const result = this.database.transaction(() => 'scope' in operation + ? this.commitTerminal(expected, next, operation) + : this.commitTransition(expected, next, operation))(); + if (isTransition && this.transitionFault === 'after_commit') { + this.transitionFault = undefined; + throw new Error('Injected crash after state/audit transaction commit'); + } + return result; + } + + async append( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + return this.database.transaction(() => { + const state = this.readState(fence); + if (!matchesTurn(state, fence, execution)) return { accepted: false as const, reason: 'turn_not_active' as const }; + return { accepted: true as const, persisted: this.record(fence, fence.turnId, execution, event) }; + })(); + } + + async appendControl( + fence: GoalSessionControlFence, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): Promise { + return this.database.transaction(() => { + const state = this.readState(fence); + if (!state || state.controllerEpoch !== fence.controllerEpoch + || state.providerBarrierIntent?.phase === 'pending' + || ['cancelling', 'terminated', 'failed'].includes(state.status)) { + return { accepted: false as const, reason: 'stale_fence' as const }; + } + return { + accepted: true as const, + persisted: this.record(fence, `#control-e${fence.controllerEpoch}`, execution, event), + }; + })(); + } + + async replay(identity: GoalSessionIdentity, afterSequence = 0): Promise { + this.assertGoalScope(identity); + const rows = this.database.prepare( + 'SELECT payload FROM goal_session_runtime_events WHERE scope = ? AND sequence > ? ORDER BY sequence', + ).all(scope(identity), afterSequence) as Array<{ payload: string }>; + return rows.map(row => JSON.parse(row.payload) as PersistedGoalSessionEvent); + } + + async listPending(identity: GoalSessionIdentity): Promise { + this.assertGoalScope(identity); + const rows = this.database.prepare( + 'SELECT payload FROM goal_session_runtime_messages WHERE scope = ? ORDER BY sequence', + ).all(scope(identity)) as Array<{ payload: string }>; + return rows.map(row => JSON.parse(row.payload) as DurableCorrectiveMessage) + .filter(message => !message.acknowledgedAt); + } + + async acknowledge( + _fence: GoalSessionFence, + _execution: GoalExecutionIdentity, + _messageId: string, + ): Promise<'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found'> { + return this.acknowledgeMessage(_fence, _execution, _messageId, false); + } + + async acknowledgeWithEvent( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + ): Promise<'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found'> { + return this.database.transaction(() => { + const state = this.readState(fence); + if (!matchesTurn(state, fence, execution)) return 'stale_fence' as const; + const message = this.readMessage(fence, messageId); + if (!message) return 'not_found' as const; + if (message.acknowledgedAt) return 'already_acknowledged' as const; + this.writeMessage({ ...message, acknowledgedAt: new Date().toISOString() }); + this.record(fence, fence.turnId, execution, { type: 'message_acknowledged', messageId }); + return 'acknowledged' as const; + })(); + } + + enqueueMessage(message: DurableCorrectiveMessage): void { + this.assertGoalScope(message); + this.database.prepare('INSERT INTO goal_session_runtime_messages(scope, message_id, sequence, payload) VALUES (?, ?, ?, ?)') + .run(scope(message), message.messageId, message.sequence, JSON.stringify(message)); + } + + async inspectContainer(identity: GoalSessionIdentity): Promise { + this.assertGoalScope(identity); + return this.fixture('container', scope(identity)) ?? { status: 'missing', reason: 'not configured' }; + } + + async inspectRepository(repository: GoalRepositoryIdentity): Promise { + return this.fixture('repository', repository.worktreePath) ?? { + ...repository, exists: false, reason: 'not configured', + }; + } + + setContainerInspection(identity: GoalSessionIdentity, inspection: GoalContainerInspection): void { + this.assertGoalScope(identity); + this.setFixture('container', scope(identity), inspection); + } + + setRepositoryInspection(repository: GoalRepositoryIdentity, inspection: GoalRepositoryInspection): void { + this.setFixture('repository', repository.worktreePath, inspection); + } + + private readState(identity: GoalSessionIdentity): GoalSessionState | null { + this.assertGoalScope(identity); + const row = this.database.prepare('SELECT payload FROM goal_session_runtime_state WHERE scope = ?') + .get(scope(identity)) as { payload: string } | undefined; + return row ? JSON.parse(row.payload) as GoalSessionState : null; + } + + private readModelChange( + identity: GoalSessionIdentity, + operationId: string, + ): GoalModelChangeHistoryRecord | undefined { + const row = this.database.prepare( + 'SELECT sequence, model, status, acknowledgement FROM goal_session_runtime_model_changes WHERE scope = ? AND operation_id = ?', + ).get(scope(identity), operationId) as { + sequence: number; model: string; status: GoalModelChangeHistoryRecord['status']; acknowledgement: string | null; + } | undefined; + return row ? { + operationId, sequence: row.sequence, model: row.model, status: row.status, + acknowledgement: row.acknowledgement + ? JSON.parse(row.acknowledgement) as GoalModelChangeAcknowledgement : undefined, + } : undefined; + } + + private acknowledgeMessage( + fence: GoalSessionFence, + execution: GoalExecutionIdentity, + messageId: string, + withEvent: boolean, + ): 'acknowledged' | 'already_acknowledged' | 'stale_fence' | 'not_found' { + return this.database.transaction(() => { + if (!matchesTurn(this.readState(fence), fence, execution)) return 'stale_fence' as const; + const message = this.readMessage(fence, messageId); + if (!message) return 'not_found' as const; + if (message.acknowledgedAt) return 'already_acknowledged' as const; + this.writeMessage({ ...message, acknowledgedAt: new Date().toISOString() }); + if (withEvent) this.record(fence, fence.turnId, execution, { type: 'message_acknowledged', messageId }); + return 'acknowledged' as const; + })(); + } + + private readMessage(identity: GoalSessionIdentity, messageId: string): DurableCorrectiveMessage | undefined { + const row = this.database.prepare('SELECT payload FROM goal_session_runtime_messages WHERE scope = ? AND message_id = ?') + .get(scope(identity), messageId) as { payload: string } | undefined; + return row ? JSON.parse(row.payload) as DurableCorrectiveMessage : undefined; + } + + private writeMessage(message: DurableCorrectiveMessage): void { + this.database.prepare('UPDATE goal_session_runtime_messages SET payload = ? WHERE scope = ? AND message_id = ?') + .run(JSON.stringify(message), scope(message), message.messageId); + } + + private commitTransition( + expected: GoalSessionState, + next: Omit, + transition: GoalSessionControlTransition, + ): GoalSessionState | null { + const current = this.readState(expected); + if (!matchesTransition(current, transition)) return null; + const identity = transitionKey(transition); + if (this.hasCommit('transition', identity)) return current; + if (current.version !== expected.version) return null; + const saved = { ...clone(next), version: current.version + 1 }; + this.writeState(current, saved); + for (const event of transition.auditEvents) { + const turnId = transition.turnScoped === true && 'turnId' in transition.fence + ? transition.fence.turnId : `#control-e${transition.fence.controllerEpoch}`; + this.record(transition.fence, turnId, transition.execution, event); + } + this.addCommit('transition', identity); + return saved; + } + + private commitTerminal( + expected: GoalSessionState, + next: Omit, + completion: GoalTerminalCommit, + ): GoalSessionState | null { + const current = this.readState(expected); + const identity = terminalKey(completion); + if (this.hasCommit('terminal', identity)) return current; + if (!current || current.version !== expected.version + || current.controllerEpoch !== completion.fence.controllerEpoch) return null; + if (completion.scope === 'turn' && !matchesTurn(current, completion.fence, completion.execution)) return null; + const saved = { ...clone(next), version: current.version + 1 }; + this.writeState(current, saved); + const turnId = completion.scope === 'turn' + ? completion.fence.turnId : `#control-e${completion.fence.controllerEpoch}`; + for (const event of completion.auditEvents) this.record(completion.fence, turnId, completion.execution, event); + this.record(completion.fence, turnId, completion.execution, completion.event); + this.addCommit('terminal', identity); + return saved; + } + + private writeState(current: GoalSessionState, saved: GoalSessionState): void { + const result = this.database.prepare('UPDATE goal_session_runtime_state SET payload = ? WHERE scope = ? AND payload = ?') + .run(JSON.stringify(saved), scope(current), JSON.stringify(current)); + if (result.changes !== 1) throw new Error('SQLite state changed inside an immediate transaction'); + } + + private record( + fence: GoalSessionControlFence, + turnId: string, + execution: GoalExecutionIdentity, + event: GoalSessionEvent, + ): PersistedGoalSessionEvent { + const row = this.database.prepare('SELECT COALESCE(MAX(sequence), 0) AS sequence FROM goal_session_runtime_events WHERE scope = ?') + .get(scope(fence)) as { sequence: number }; + const persisted: PersistedGoalSessionEvent = { + ...fence, turnId, ...execution, sequence: row.sequence + 1, + recordedAt: new Date().toISOString(), event: clone(sanitizeGoalSessionEvent(event)), + }; + this.database.prepare('INSERT INTO goal_session_runtime_events(scope, sequence, payload) VALUES (?, ?, ?)') + .run(scope(fence), persisted.sequence, JSON.stringify(persisted)); + return persisted; + } + + private fixture(kind: string, identity: string): T | undefined { + const row = this.database.prepare('SELECT payload FROM goal_session_runtime_fixtures WHERE kind = ? AND identity = ?') + .get(kind, identity) as { payload: string } | undefined; + return row ? JSON.parse(row.payload) as T : undefined; + } + + private setFixture(kind: string, identity: string, value: unknown): void { + this.database.prepare( + 'INSERT INTO goal_session_runtime_fixtures(kind, identity, payload) VALUES (?, ?, ?) ' + + 'ON CONFLICT(kind, identity) DO UPDATE SET payload = excluded.payload', + ).run(kind, identity, JSON.stringify(value)); + } + + private hasCommit(kind: string, identity: string): boolean { + return Boolean(this.database.prepare('SELECT 1 FROM goal_session_runtime_commits WHERE kind = ? AND identity = ?').get(kind, identity)); + } + + private addCommit(kind: string, identity: string): void { + this.database.prepare('INSERT INTO goal_session_runtime_commits(kind, identity) VALUES (?, ?)').run(kind, identity); + } + + private assertGoalScope(identity: GoalSessionIdentity): void { + const owner = this.database.prepare( + 'SELECT goal_id FROM goal_session_runtime_owners WHERE session_id = ?', + ).get(identity.sessionId) as { goal_id: string } | undefined; + if (owner && owner.goal_id !== identity.goalId) throw new GoalSessionScopeError(); + } +} + +function matchesTurn( + state: GoalSessionState | null, + fence: GoalSessionFence, + execution: GoalExecutionIdentity, +): state is GoalSessionState { + return Boolean(state && state.controllerEpoch === fence.controllerEpoch + && state.providerBarrierIntent?.phase !== 'pending' + && !['cancelling', 'terminated', 'failed'].includes(state.status) + && state.activeTurn?.turnId === fence.turnId + && state.activeTurn.executionId === execution.executionId + && state.activeTurn.attemptId === execution.attemptId + && !['completed', 'cancelled', 'failed'].includes(state.activeTurn.status)); +} + +function matchesTransition( + state: GoalSessionState | null, + transition: GoalSessionControlTransition, +): state is GoalSessionState { + if (!state || state.controllerEpoch !== transition.fence.controllerEpoch + || state.providerBarrierIntent?.phase === 'pending' + || ['cancelling', 'terminated', 'failed'].includes(state.status)) return false; + return transition.turnScoped !== true + || ('turnId' in transition.fence && matchesTurn(state, transition.fence, transition.execution)); +} + +function transitionKey(value: GoalSessionControlTransition): string { + return JSON.stringify([scope(value.fence), value.fence.controllerEpoch, + value.turnScoped === true && 'turnId' in value.fence ? value.fence.turnId : null, + value.execution.executionId, value.execution.attemptId, value.transitionId]); +} + +function terminalKey(value: GoalTerminalCommit): string { + return JSON.stringify([value.scope, scope(value.fence), value.fence.controllerEpoch, + value.scope === 'turn' ? value.fence.turnId : null, + value.execution.executionId, value.execution.attemptId]); +} diff --git a/packages/core/test/codexAppServer0146LiveContract.test.ts b/packages/core/test/codexAppServer0146LiveContract.test.ts new file mode 100644 index 000000000..c0895f26d --- /dev/null +++ b/packages/core/test/codexAppServer0146LiveContract.test.ts @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import readline from 'node:readline'; +import { test } from 'node:test'; + +type RpcResponse = { id: number; result?: Record; error?: { code: number; message: string } }; + +function codexVersion(): string | undefined { + try { return execFileSync('codex', ['--version'], { encoding: 'utf8' }).trim(); } + catch { return undefined; } +} + +test('generated and live Codex 0.146 contract accepts workspace-write and rejects workspaceWrite', async t => { + const version = codexVersion(); + if (!version) return t.skip('codex binary is not installed in this validation environment'); + assert.equal(version, 'codex-cli 0.146.0'); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-0146-contract-')); + const generated = path.join(root, 'generated'); + const codexHome = path.join(root, 'home'); + fs.mkdirSync(codexHome, { recursive: true }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + execFileSync('codex', ['app-server', 'generate-ts', '--experimental', '--out', generated]); + const sandbox = fs.readFileSync(path.join(generated, 'v2', 'SandboxMode.ts'), 'utf8'); + const source = fs.readFileSync(path.join(generated, 'v2', 'SessionSource.ts'), 'utf8'); + const response = fs.readFileSync(path.join(generated, 'v2', 'ThreadStartResponse.ts'), 'utf8'); + assert.match(sandbox, /"workspace-write"/); + assert.doesNotMatch(sandbox, /"workspaceWrite"/); + for (const variant of ['appServer', 'subAgent', 'custom', 'unknown']) assert.match(source, new RegExp(variant)); + for (const field of ['thread', 'modelProvider', 'runtimeWorkspaceRoots', 'instructionSources']) { + assert.match(response, new RegExp(field)); + } + + const child = spawn('codex', ['app-server'], { + cwd: process.cwd(), env: { ...process.env, CODEX_HOME: codexHome }, stdio: ['pipe', 'pipe', 'pipe'], + }); + t.after(() => { if (child.exitCode === null) child.kill('SIGTERM'); }); + const pending = new Map void>(); + readline.createInterface({ input: child.stdout }).on('line', line => { + const value = JSON.parse(line) as RpcResponse; + if (typeof value.id === 'number') pending.get(value.id)?.(value); + }); + const request = (id: number, method: string, params: Record): Promise => { + const result = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`live ${method} timed out`)), 10_000); + pending.set(id, value => { clearTimeout(timer); pending.delete(id); resolve(value); }); + }); + child.stdin.write(`${JSON.stringify({ id, method, params })}\n`); + return result; + }; + const initialized = await request(1, 'initialize', { + clientInfo: { name: 'propr_goal_runtime', title: 'ProPR Goal Runtime', version: '0.146.0' }, + capabilities: { experimentalApi: false, requestAttestation: false }, + }); + assert.equal(initialized.result?.codexHome, codexHome); + assert.match(String(initialized.result?.userAgent), /^propr_goal_runtime\/0\.146\.0 \(/); + child.stdin.write(`${JSON.stringify({ method: 'initialized' })}\n`); + + const common = { model: 'gpt-5.6-sol', cwd: process.cwd(), approvalPolicy: 'never' }; + const rejected = await request(2, 'thread/start', { ...common, sandbox: 'workspaceWrite' }); + assert.equal(rejected.error?.code, -32600); + const accepted = await request(3, 'thread/start', { ...common, sandbox: 'workspace-write' }); + assert.equal(accepted.error, undefined); + assert.equal(typeof (accepted.result?.thread as Record)?.id, 'string'); +}); diff --git a/packages/core/test/fixtures/fake-pending-open-docker.mjs b/packages/core/test/fixtures/fake-pending-open-docker.mjs new file mode 100755 index 000000000..e20e91aec --- /dev/null +++ b/packages/core/test/fixtures/fake-pending-open-docker.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node +import fs from 'node:fs'; + +const statePath = process.env.GOAL_PENDING_OPEN_STATE; +const logPath = process.env.GOAL_PENDING_OPEN_LOG; +const expected = JSON.parse(process.env.GOAL_PENDING_OPEN_LABELS ?? '{}'); +const args = process.argv.slice(2); +if (!statePath || !logPath) process.exit(2); +fs.appendFileSync(logPath, `${JSON.stringify(args)}\n`); + +if (args[0] === 'ps') { + const filters = args.flatMap((argument, index) => argument === '--filter' ? [args[index + 1]] : []); + const matches = Object.entries(expected).every(([name, value]) => filters.includes(`label=${name}=${value}`)); + if (matches && fs.existsSync(statePath)) process.stdout.write(`${fs.readFileSync(statePath, 'utf8').trim()}\n`); + process.exit(0); +} +if (args[0] === 'rm' && args[1] === '-f' && args[2]) { + try { fs.unlinkSync(statePath); } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + process.exit(0); +} +process.exit(2); diff --git a/packages/core/test/goalContainerHardening.test.ts b/packages/core/test/goalContainerHardening.test.ts new file mode 100644 index 000000000..a11681648 --- /dev/null +++ b/packages/core/test/goalContainerHardening.test.ts @@ -0,0 +1,689 @@ +import assert from 'node:assert/strict'; +import * as actualChildProcess from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { mock, test } from 'node:test'; +import Database from 'better-sqlite3'; +import type { GoalSessionAdapter } from '../src/agents/goalSession/contract.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { createSqliteGoalSessionRuntimePorts } from '../src/agents/goalSession/SqliteGoalSessionControlDomain.js'; +import { createProductionSchema, recovery, seedAuthoritativeGoal } from './productionGoalSessionTestSupport.js'; + +const spawnCalls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; +let stdinHandler: ((data: string) => void) | undefined; +const outputStream = () => Object.assign(new EventEmitter(), { + pause: mock.fn(), + resume: mock.fn(), +}); +const child = Object.assign(new EventEmitter(), { + stdout: outputStream(), + stderr: outputStream(), + stdin: { destroyed: false, writableEnded: false, write(data: string, cb: (e?: Error | null) => void) { stdinHandler?.(data); cb(); return true; }, end() { this.writableEnded = true; } }, + exitCode: null as number | null, + kill: mock.fn(() => true), +}); + +await mock.module('child_process', { + namedExports: { + ...actualChildProcess, + spawn: mock.fn((_command: string, args: string[], options?: { env?: NodeJS.ProcessEnv }) => { + spawnCalls.push({ args, env: options?.env }); + return child; + }), + execFileSync: mock.fn(), + }, +}); + +const { GoalContainerSupervisor, buildGoalContainerLayout } = await import('../src/agents/goalSession/GoalContainerSupervisor.js'); +const { createSupervisedCodexAppServerFactory } = await import('../src/agents/goalSession/supervisedCodexOpenFactory.js'); +const { GoalSessionSupervisor } = await import('../src/agents/goalSession/GoalSessionSupervisor.js'); +type EventSink = ConstructorParameters[1]; + +const events = { append: async () => ({ accepted: true }), appendControl: async () => ({ accepted: true }), replay: async () => [] } as unknown as EventSink; +const idBits = { goalId: 'g', sessionId: 's', controllerEpoch: 1, turnId: 't', executionId: 'e', attemptId: 'a' }; +const approvedWorktree = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-worktree-')); +const approvedCredential = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'goal-credential-')), 'token'); +fs.writeFileSync(approvedCredential, 'secret'); +const isolation = { + environmentKeys: ['OPENAI_API_KEY'], + worktreePaths: [approvedWorktree], + providerHomeTargets: ['/home/node/.codex'], + credentialMounts: [{ source: approvedCredential, target: '/home/node/.creds' }], +}; +const firstEffects = { + start: async ( + _fence: unknown, + _stage: unknown, + effect: () => { completion: Promise }, + ): Promise => effect().completion, +}; + +function baseRequest() { + return { + ...idBits, + operationFence: { + goalId: idBits.goalId, sessionId: idBits.sessionId, controllerEpoch: idBits.controllerEpoch, + turnId: idBits.turnId, executionId: idBits.executionId, attemptId: idBits.attemptId, + generation: 1, operationId: 'turn-operation', kind: 'turn' as const, + }, + image: 'propr/agent:test', + command: ['agent-command'], + worktreePath: approvedWorktree, + worktreeFingerprint: 'fingerprint-one', + providerHomeTarget: '/home/node/.codex', + }; +} + +function createSupervisor(base: string, policy = isolation): InstanceType { + return new GoalContainerSupervisor(base, events, undefined, { isolation: policy, providerFirstEffects: firstEffects }); +} + +async function waitForFile(filePath: string): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (fs.existsSync(filePath) && fs.statSync(filePath).size > 0) return; + await new Promise(resolve => setTimeout(resolve, 5)); + } + throw new Error(`Timed out waiting for ${filePath}`); +} + +test('start passes env names only and never leaks secret values into argv', async () => { + spawnCalls.length = 0; + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-hard-')); + const supervisor = createSupervisor(base); + await supervisor.start({ + ...baseRequest(), + environment: { OPENAI_API_KEY: 'super-secret-value' }, + credentialMounts: [{ source: approvedCredential, target: '/home/node/.creds' }], + }); + const args = spawnCalls[0].args; + const envIndex = args.indexOf('--env'); + assert.equal(args[envIndex + 1], 'OPENAI_API_KEY'); + assert.ok(!args.some(arg => arg.includes('super-secret-value')), 'secret value must not appear in argv'); + assert.ok(args.includes(`type=bind,src=${approvedCredential},dst=/home/node/.creds,readonly`)); + assert.deepEqual(spawnCalls[0].env, { OPENAI_API_KEY: 'super-secret-value' }); +}); + +test('layout logPath is an actually used goal-scoped durable output sink', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-log-')); + const supervisor = createSupervisor(base); + const { layout } = await supervisor.start(baseRequest()); + child.stdout.emit('data', Buffer.from('auditable output\n')); + await waitForFile(layout.logPath); + + const records = fs.readFileSync(layout.logPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + assert.ok(records.some(record => record.channel === 'stdout' + && record.attemptId === idBits.attemptId + && record.data === 'auditable output\n')); + assert.ok(fs.statSync(layout.logPath).size <= 8 * 1024 * 1024); +}); + +test('goal JSONL persists only public output fields from a secret-poisoned start request', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-log-secret-')); + const supervisor = createSupervisor(base); + const poisoned = { + ...baseRequest(), + command: ['provider-command', '--token', 'command-secret-value'], + environment: { OPENAI_API_KEY: 'environment-secret-value' }, + credentialMounts: [{ source: approvedCredential, target: '/home/node/.creds' }], + taskId: 'private-task-secret', + }; + const { layout } = await supervisor.start(poisoned); + child.stderr.emit('data', Buffer.from('public diagnostic\n')); + await waitForFile(layout.logPath); + + const bytes = fs.readFileSync(layout.logPath, 'utf8'); + for (const secret of [ + 'command-secret-value', + 'environment-secret-value', + approvedCredential, + approvedWorktree, + 'private-task-secret', + 'provider-command', + ]) assert.ok(!bytes.includes(secret), `JSONL leaked ${secret}`); + const record = JSON.parse(bytes.trim().split('\n')[0]); + assert.deepEqual(Object.keys(record).sort(), [ + 'attemptId', 'channel', 'controllerEpoch', 'data', 'executionId', 'goalId', + 'recordedAt', 'sequence', 'sessionId', 'truncated', 'turnId', 'worktreeFingerprint', + ]); + assert.equal(record.executionId, idBits.executionId); + assert.equal(record.attemptId, idBits.attemptId); + assert.equal(record.data, 'public diagnostic\n'); +}); + +test('raw durable event DTOs and replay bytes exclude every poisoned start-request field', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-event-secret-')); + const persistence = new InMemoryGoalSessionPorts(); + const ids = { + goalId: 'raw-goal', sessionId: 'raw-session', controllerEpoch: 7, + turnId: 'raw-turn', executionId: 'raw-execution', attemptId: 'raw-attempt', + }; + const timestamp = new Date().toISOString(); + await persistence.create({ + goalId: ids.goalId, + sessionId: ids.sessionId, + provider: 'raw-provider', + providerSessionId: 'raw-provider-session', + recoveryMetadata: { checkpoint: 'public' }, + controllerEpoch: ids.controllerEpoch, + status: 'running', + activeTurn: { + executionId: ids.executionId, + attemptId: ids.attemptId, + turnId: ids.turnId, + executionEpoch: ids.controllerEpoch, + objective: 'public objective', + requestedModel: 'public-model', + repository: { repository: 'integry/propr', worktreePath: approvedWorktree, branch: 'test' }, + status: 'running', + }, + completedTurnIds: [], + createdAt: timestamp, + updatedAt: timestamp, + }); + let delivered: unknown; + const runtime = persistence.asRuntimePorts(); + const capturingEvents: EventSink = { + append: async (publicFence, publicExecution, event) => { + delivered = structuredClone({ publicFence, publicExecution, event }); + return runtime.events.append(publicFence, publicExecution, event); + }, + appendControl: (publicFence, publicExecution, event) => + runtime.events.appendControl(publicFence, publicExecution, event), + replay: (eventIdentity, afterSequence) => runtime.events.replay(eventIdentity, afterSequence), + }; + const supervisor = new GoalContainerSupervisor(base, capturingEvents, undefined, { + isolation, providerFirstEffects: firstEffects, + }); + const secrets = [ + 'poison-environment-secret', 'poison-command-secret', 'poison-task-secret', + 'poison-excess-secret', approvedCredential, approvedWorktree, + ]; + await supervisor.start({ + ...baseRequest(), + ...ids, + operationFence: { + ...baseRequest().operationFence, ...ids, + }, + command: ['provider', '--secret', secrets[1]], + environment: { OPENAI_API_KEY: secrets[0] }, + credentialMounts: [{ source: approvedCredential, target: '/home/node/.creds' }], + taskId: secrets[2], + arbitraryExcess: secrets[3], + } as ReturnType & typeof ids & { arbitraryExcess: string; taskId: string }); + child.stdout.emit('data', Buffer.from('public raw output')); + for (let attempt = 0; attempt < 100 && (await persistence.replay(ids)).length === 0; attempt += 1) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + + const replayed = await persistence.replay(ids); + assert.equal(replayed.length, 1); + assert.deepEqual(Object.keys(replayed[0]).sort(), [ + 'attemptId', 'controllerEpoch', 'event', 'executionId', 'goalId', 'recordedAt', + 'sequence', 'sessionId', 'turnId', + ]); + assert.deepEqual(Object.keys(replayed[0].event).sort(), ['channel', 'data', 'type']); + const rawBytes = JSON.stringify({ delivered, replayed }); + for (const secret of secrets) assert.ok(!rawBytes.includes(secret), `raw event persistence leaked ${secret}`); +}); + +test('layout log sink truncates deterministically at its auditable byte bound', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-log-bound-')); + const supervisor = createSupervisor(base); + const { layout, execution } = await supervisor.start({ + ...baseRequest(), + goalId: 'bounded-log-goal', + sessionId: 'bounded-log-session', + operationFence: { + ...baseRequest().operationFence, + goalId: 'bounded-log-goal', sessionId: 'bounded-log-session', + }, + }); + child.stdout.emit('data', Buffer.alloc(8 * 1024 * 1024, 'x')); + child.emit('close', 0); + await execution.completion; + + const size = fs.statSync(layout.logPath).size; + assert.ok(size > 0); + assert.ok(size <= 8 * 1024 * 1024); + const records = fs.readFileSync(layout.logPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + assert.equal(records.at(-1)?.truncated, true); + assert.ok(records.every(record => record.goalId === 'bounded-log-goal' + && record.sessionId === 'bounded-log-session' + && record.attemptId === idBits.attemptId)); +}); + +test('start rejects provider homes that shadow reserved or non-provider paths', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-hard-')); + const supervisor = createSupervisor(base); + await assert.rejects(supervisor.start({ ...baseRequest(), providerHomeTarget: '/workspace' }), /workspace/); + await assert.rejects(supervisor.start({ ...baseRequest(), providerHomeTarget: '/' }), /reserved/); + await assert.rejects(supervisor.start({ ...baseRequest(), providerHomeTarget: '/etc/agent' }), /provider-owned/); +}); + +test('start supports an explicitly configured read-only Codex credential file at its native target', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-hard-')); + const supervisor = createSupervisor(base, { + ...isolation, + credentialMounts: [{ source: approvedCredential, target: '/home/node/.codex/creds' }], + }); + await supervisor.start({ + ...baseRequest(), credentialMounts: [{ + provider: 'codex', source: approvedCredential, target: '/home/node/.codex/creds', + }], + }); + assert.ok(spawnCalls.at(-1)?.args.includes( + `type=bind,src=${approvedCredential},dst=/home/node/.codex/creds,readonly`, + )); +}); + +test('separate credential-file ingress supports explicit Claude, Codex, and Antigravity native auth files', async () => { + const profiles = [ + { provider: 'claude' as const, home: '/home/node/.claude', target: '/home/node/.claude.json' }, + { provider: 'codex' as const, home: '/home/node/.codex', target: '/home/node/.codex/auth.json' }, + { provider: 'antigravity' as const, home: '/home/node/.gemini', target: '/home/node/.gemini/oauth_creds.json' }, + ]; + for (const profile of profiles) { + const base = fs.mkdtempSync(path.join(os.tmpdir(), `goal-${profile.provider}-auth-`)); + const mount = { provider: profile.provider, source: approvedCredential, target: profile.target }; + const supervisor = createSupervisor(base, { + ...isolation, providerHomeTargets: [profile.home], credentialMounts: [mount], + }); + await supervisor.start({ + ...baseRequest(), providerHomeTarget: profile.home, credentialMounts: [mount], + }); + assert.ok(spawnCalls.at(-1)?.args.includes( + `type=bind,src=${approvedCredential},dst=${profile.target},readonly`, + )); + } +}); + +test('adapter output observes the exact durable mixed-channel queue with backpressure and unsubscribe', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-adapter-output-')); + const durable: Array<{ channel: string; data: string }> = []; + const sink = { + ...events, + append: async (_fence: unknown, _execution: unknown, event: { channel: string; data: string }) => { + durable.push({ channel: event.channel, data: event.data }); + return { accepted: true as const }; + }, + } as unknown as EventSink; + let releaseFirst!: () => void; + const firstGate = new Promise(resolve => { releaseFirst = resolve; }); + const observed: Array<{ sequence: number; channel: string; data: string }> = []; + const supervisor = new GoalContainerSupervisor(base, sink, undefined, { + isolation, providerFirstEffects: firstEffects, + }); + await supervisor.start({ + ...baseRequest(), + outputObserver: { + next: async output => { + assert.equal(durable.length, output.sequence, 'durability precedes adapter delivery in the same queue'); + observed.push({ sequence: output.sequence, channel: output.channel, data: output.data }); + if (output.sequence === 1) await firstGate; + if (output.sequence === 2) return 'unsubscribe'; + }, + }, + }); + child.stdout.emit('data', Buffer.from('first')); + child.stderr.emit('data', Buffer.from('second')); + for (let attempt = 0; attempt < 100 && observed.length === 0; attempt += 1) { + await new Promise(resolve => setImmediate(resolve)); + } + assert.deepEqual(observed.map(output => output.data), ['first']); + assert.equal(durable.length, 1, 'the supervised source remains backpressured behind the adapter'); + releaseFirst(); + for (let attempt = 0; attempt < 100 && observed.length < 2; attempt += 1) { + await new Promise(resolve => setImmediate(resolve)); + } + assert.deepEqual(observed, [ + { sequence: 1, channel: 'stdout', data: 'first' }, + { sequence: 2, channel: 'stderr', data: 'second' }, + ]); + child.stdout.emit('data', Buffer.from('durable-only')); + for (let attempt = 0; attempt < 100 && durable.length < 3; attempt += 1) { + await new Promise(resolve => setImmediate(resolve)); + } + assert.equal(observed.length, 2, 'unsubscribe does not bypass or tail durable output'); + assert.deepEqual(durable.map(output => output.data), ['first', 'second', 'durable-only']); +}); + +test('protocol observer receives parseable secret bytes while every durable surface is redacted', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-protocol-secret-')); + const durable: string[] = []; + const observed: string[] = []; + const sink = { + ...events, + append: async (_fence: unknown, _execution: unknown, event: { data: string }) => { + durable.push(event.data); + return { accepted: true as const }; + }, + } as unknown as EventSink; + const supervisor = new GoalContainerSupervisor(base, sink, undefined, { + isolation, providerFirstEffects: firstEffects, + }); + const { layout } = await supervisor.start({ + ...baseRequest(), + outputObserver: { next: output => { observed.push(output.data); } }, + }); + const protocol = '{"method":"initialize","token":"HOSTILE-PROTOCOL-SECRET"}\n'; + child.stdout.emit('data', Buffer.from(protocol)); + for (let attempt = 0; attempt < 100 && observed.length === 0; attempt += 1) { + await new Promise(resolve => setImmediate(resolve)); + } + assert.deepEqual(JSON.parse(observed.join('').trim()), { + method: 'initialize', token: 'HOSTILE-PROTOCOL-SECRET', + }); + assert.deepEqual(durable, ['[redacted output]']); + const serialized = `${JSON.stringify(durable)}\n${fs.readFileSync(layout.logPath, 'utf8')}`; + assert.doesNotMatch(serialized, /HOSTILE-PROTOCOL-SECRET/); +}); + +test('control-scoped eager open container has exact open labels and no invented turn label', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-open-container-')); + const supervisor = new GoalContainerSupervisor(base, events, undefined, { + isolation, providerFirstEffects: firstEffects, + }); + await supervisor.startOpen({ + goalId: 'open-goal', sessionId: 'open-session', controllerEpoch: 4, + executionId: 'open-execution', attemptId: 'open-attempt', deterministicOpenKey: 'open-key-4', + operationFence: { + goalId: 'open-goal', sessionId: 'open-session', controllerEpoch: 4, + executionId: 'open-execution', attemptId: 'open-attempt', generation: 7, + operationId: 'open-attempt', kind: 'open', + }, + image: 'provider-image', command: ['codex', 'app-server', '--stdio'], + worktreePath: approvedWorktree, worktreeFingerprint: 'open-fingerprint', + providerHomeTarget: '/home/node/.codex', + }); + const args = spawnCalls.at(-1)!.args; + assert.ok(args.includes('propr.goal.scope=open')); + assert.ok(args.includes('propr.goal.open-key=open-key-4')); + assert.equal(args.some(argument => argument.startsWith('propr.goal.turn=')), false); + assert.ok(args.includes('/workspace')); +}); + +test('credential targets reject descendants of proc, sys, and dev even when allow-listed', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-pseudo-fs-')); + const targets = ['/proc/self/fd/9', '/sys/kernel/credential', '/dev/shm/credential']; + const supervisor = createSupervisor(base, { + ...isolation, + credentialMounts: targets.map(target => ({ source: approvedCredential, target })), + }); + for (const target of targets) { + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: approvedCredential, target }] }), + /broad or sensitive container path/, + ); + } +}); + +test('credential targets reject pseudo-filesystem traversal and symlink-equivalent aliases', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-pseudo-alias-')); + const targets = [ + '/safe/../proc/self/fd/9', + '/sys//kernel/credential', + '/dev/./shm/credential', + '/dev/fd/9', + '/proc/self/root/dev/null', + ]; + const supervisor = createSupervisor(base, { + ...isolation, + credentialMounts: targets.map(target => ({ source: approvedCredential, target })), + }); + for (const target of targets) { + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: approvedCredential, target }] }), + /canonical|broad or sensitive container path/, + ); + } +}); + +test('cleanTerminalSession removes a real goal directory but refuses a symlink escape', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-clean-')); + const supervisor = createSupervisor(base); + fs.mkdirSync(path.join(base, 'goals'), { recursive: true }); + const past = new Date(0); + const future = new Date(Date.now() + 10 * 24 * 60 * 60 * 1000); + + // Happy path: a real, goal-scoped directory is removed once retention lapses. + const real = buildGoalContainerLayout(base, idBits); + fs.mkdirSync(real.sessionRoot, { recursive: true }); + fs.writeFileSync(path.join(real.sessionRoot, 'state.json'), '{}'); + assert.equal(await supervisor.cleanTerminalSession(real, past, 'succeeded', future), true); + assert.equal(fs.existsSync(real.sessionRoot), false); + + // Escape attempt: a symlinked session root pointing outside the goals tree. + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-outside-')); + fs.writeFileSync(path.join(outside, 'keep.txt'), 'precious'); + const escape = buildGoalContainerLayout(base, { ...idBits, goalId: 'escape-goal' }); + fs.symlinkSync(outside, escape.sessionRoot); + await assert.rejects( + supervisor.cleanTerminalSession(escape, past, 'succeeded', future), + /symlinked goal session directory/, + ); + assert.equal(fs.existsSync(path.join(outside, 'keep.txt')), true); +}); + +test('cleanTerminalSession refuses an in-tree symlink to a sibling goal directory', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-sibling-')); + const supervisor = new GoalContainerSupervisor(base, events, undefined, { providerFirstEffects: firstEffects }); + fs.mkdirSync(path.join(base, 'goals'), { recursive: true }); + const past = new Date(0); + const future = new Date(Date.now() + 10 * 24 * 60 * 60 * 1000); + + // A real sibling goal directory that must survive cleanup of another goal. + const sibling = buildGoalContainerLayout(base, { ...idBits, goalId: 'sibling-goal' }); + fs.mkdirSync(sibling.sessionRoot, { recursive: true }); + fs.writeFileSync(path.join(sibling.sessionRoot, 'sibling-state.json'), '{}'); + + // The cleaned goal's own session root is a symlink to the sibling's real + // directory; dirname(resolvedRoot) still equals the goals dir, so the old + // parent-only check would have deleted the sibling. Identity check rejects it. + const attacker = buildGoalContainerLayout(base, { ...idBits, goalId: 'attacker-goal' }); + fs.symlinkSync(sibling.sessionRoot, attacker.sessionRoot); + await assert.rejects( + supervisor.cleanTerminalSession(attacker, past, 'succeeded', future), + /symlinked goal session directory/, + ); + assert.equal(fs.existsSync(path.join(sibling.sessionRoot, 'sibling-state.json')), true); +}); + +test('buildGoalContainerLayout keeps the log path inside the goal log directory', () => { + // Separator/traversal-laden execution and attempt ids must not escape logs/. + const layout = buildGoalContainerLayout('/var/lib/propr', { + ...idBits, + executionId: '../../../../etc/cron.d/evil', + attemptId: 'a/b/../..', + }); + const logDir = path.join(layout.sessionRoot, 'logs'); + assert.equal(path.dirname(path.resolve(layout.logPath)), path.resolve(logDir)); + assert.ok(layout.logPath.startsWith(`${logDir}/`)); + assert.ok(!layout.logPath.includes('..')); + assert.ok(!layout.logPath.includes('etc/cron.d')); +}); + +test('start rejects bind-mount fields that could inject Docker --mount options', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-inject-')); + const supervisor = createSupervisor(base); + await assert.rejects( + supervisor.start({ ...baseRequest(), worktreePath: '/tmp/wt,readonly,bind-propagation=rshared' }), + /inject Docker --mount options/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: '/host/creds,readonly', target: '/home/node/.creds' }] }), + /inject Docker --mount options/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: approvedCredential, target: '/home/node/.creds,dst=/etc' }] }), + /inject Docker --mount options/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), providerHomeTarget: '/home/node/.codex,type=volume' }), + /inject Docker --mount options/, + ); +}); + +test('start blocks host-controlled environment aliases even when configured', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-env-')); + const supervisor = createSupervisor(base, { + ...isolation, + environmentKeys: ['DOCKER_HOST', 'docker_host', 'LD_PRELOAD'], + }); + await assert.rejects(supervisor.start({ ...baseRequest(), environment: { DOCKER_HOST: 'tcp://attacker' } }), /host-controlled/); + await assert.rejects(supervisor.start({ ...baseRequest(), environment: { docker_host: 'tcp://attacker' } }), /host-controlled/); + await assert.rejects(supervisor.start({ ...baseRequest(), environment: { LD_PRELOAD: '/tmp/evil.so' } }), /host-controlled/); +}); + +test('start rejects unapproved, broad, sensitive, and symlink-aliased mount sources', async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-mounts-')); + const outsideWorktree = fs.mkdtempSync(path.join(os.tmpdir(), 'other-worktree-')); + const worktreeAlias = path.join(os.tmpdir(), `worktree-alias-${process.pid}`); + fs.symlinkSync(approvedWorktree, worktreeAlias); + const sensitiveDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mount-owner-')); + const sshDir = path.join(sensitiveDir, '.ssh'); + fs.mkdirSync(sshDir); + const sshKey = path.join(sshDir, 'id_rsa'); + fs.writeFileSync(sshKey, 'private'); + const supervisor = createSupervisor(base, { + ...isolation, + credentialMounts: [ + { source: '/', target: '/home/node/key' }, + { source: sshKey, target: '/home/node/key' }, + { source: approvedCredential, target: '/run/docker.sock' }, + ], + worktreePaths: [approvedWorktree, worktreeAlias], + }); + + await assert.rejects(supervisor.start({ ...baseRequest(), worktreePath: outsideWorktree }), /not explicitly allow-listed/); + await assert.rejects(supervisor.start({ ...baseRequest(), worktreePath: worktreeAlias }), /symlink alias/); + await assert.rejects( + supervisor.start({ + ...baseRequest(), + worktreePath: `${path.dirname(approvedWorktree)}/alias/../${path.basename(approvedWorktree)}`, + }), + /traversal aliases/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), providerHomeTarget: '/home/node/alias/../.codex' }), + /traversal aliases/, + ); + await assert.rejects( + supervisor.start({ + ...baseRequest(), + credentialMounts: [{ source: approvedCredential, target: '/home/node/alias/../.creds' }], + }), + /traversal aliases/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: '/', target: '/home/node/key' }] }), + /broad or sensitive/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: sshKey, target: '/home/node/key' }] }), + /broad or sensitive/, + ); + await assert.rejects( + supervisor.start({ ...baseRequest(), credentialMounts: [{ source: approvedCredential, target: '/run/docker.sock' }] }), + /broad or sensitive/, + ); +}); + +test('production Codex factory composes claimed supervisor, duplex, and exact App Server open', async t => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-factory-open-')); + const filename = path.join(base, 'control.sqlite'); + await createProductionSchema(filename); + const supervisorDatabase = new Database(filename); + const containerDatabase = new Database(filename); + seedAuthoritativeGoal(supervisorDatabase, { goalId: 'factory-goal', agent: 'codex', model: 'gpt-5.6-sol' }); + const runtime = createSqliteGoalSessionRuntimePorts(supervisorDatabase, recovery); + const containerRuntime = createSqliteGoalSessionRuntimePorts(containerDatabase, recovery); + const containers = new GoalContainerSupervisor(base, containerRuntime.events, undefined, { + isolation: { + environmentKeys: [], worktreePaths: [approvedWorktree], + providerHomeTargets: ['/home/node/.codex'], credentialMounts: [], + }, + providerFirstEffects: containerRuntime.providerFirstEffects, + }); + const repository = { + repository: 'integry/propr', worktreePath: approvedWorktree, branch: 'factory-open', headSha: 'abcdef', + }; + const factory = createSupervisedCodexAppServerFactory(containers, { + repository, worktreeFingerprint: 'factory-fingerprint', image: 'codex-provider:test', + }); + const adapter: GoalSessionAdapter = { + provider: 'codex', + capabilities: { + nativeSessionId: 'eager', steering: 'active_turn', pause: 'after_turn', modelChange: 'next_turn', + }, + supportsDeterministicOpen: true, + publishOperationBarrier: async () => undefined, + openSession: request => factory.open(request), + beginTurn: async function* () { yield { type: 'completion', outcome: 'succeeded' }; }, + resumeSession: async (_request, snapshot) => snapshot, + requestModelChange: async request => ({ requestedModel: request.model, appliesAt: 'next_turn' }), + cancel: async () => undefined, + cancelPending: (request, pending) => factory.cancelPending(request, pending), + reconcile: async () => ({ outcome: 'failed', reason: 'unused' }), + }; + child.exitCode = null; + child.stdin.writableEnded = false; + stdinHandler = data => { + const request = JSON.parse(data) as { id?: string; method: string }; + if (!request.id) return; + let result: Record; + if (request.method === 'initialize') result = { + userAgent: 'propr_goal_runtime/0.146.0 (Linux; x86_64) factory', + codexHome: '/home/node/.codex', platformFamily: 'unix', platformOs: 'linux', + }; + else if (request.method === 'model/list') result = { + data: [{ id: 'gpt-5.6-sol', model: 'gpt-5.6-sol' }], nextCursor: null, + }; + else result = exactFactoryThreadResponse(); + setImmediate(() => { + child.stdout.emit('data', Buffer.from(`${JSON.stringify({ id: request.id, result })}\n`)); + if (request.method === 'thread/start') { + child.exitCode = 0; + child.emit('close', 0); + } + }); + }; + t.after(() => { + stdinHandler = undefined; child.exitCode = null; child.stdin.writableEnded = false; + supervisorDatabase.close(); containerDatabase.close(); + }); + + const supervisor = new GoalSessionSupervisor(adapter, runtime); + const opened = await supervisor.openSession({ + goalId: 'factory-goal', sessionId: 'factory-session', provider: 'codex', controllerEpoch: 1, + supervisedOpen: factory.plan, + }); + assert.equal(opened.status, 'idle'); + assert.equal(opened.providerSessionId, 'factory-thread'); + const args = spawnCalls.at(-1)!.args; + assert.ok(args.includes('/workspace')); + assert.ok(args.includes('propr.goal.scope=open')); + assert.ok(args.some(value => value.startsWith('propr.goal.operation-generation='))); + assert.ok(args.some(value => value.startsWith('propr.goal.operation-id='))); +}); + +function exactFactoryThreadResponse(): Record { + return { + thread: { + id: 'factory-thread', extra: null, sessionId: 'factory-session-native', forkedFromId: null, + parentThreadId: null, preview: '', ephemeral: false, isPinned: false, + historyMode: 'paginated', modelProvider: 'openai', createdAt: 1, updatedAt: 1, + recencyAt: 1, status: { type: 'idle' }, path: null, cwd: '/workspace', cliVersion: '0.146.0', + source: 'appServer', canAcceptDirectInput: true, threadSource: null, agentNickname: null, + agentRole: null, gitInfo: null, name: null, turns: [], + }, + model: 'gpt-5.6-sol', modelProvider: 'openai', serviceTier: null, cwd: '/workspace', + runtimeWorkspaceRoots: ['/workspace'], instructionSources: [], approvalPolicy: 'never', + approvalsReviewer: 'user', sandbox: { + type: 'workspaceWrite', writableRoots: ['/workspace'], networkAccess: false, + excludeTmpdirEnvVar: false, excludeSlashTmp: false, + }, + activePermissionProfile: null, reasoningEffort: null, multiAgentMode: 'explicitRequestOnly', + }; +} diff --git a/packages/core/test/goalContainerSupervisor.test.ts b/packages/core/test/goalContainerSupervisor.test.ts new file mode 100644 index 000000000..9fca130fb --- /dev/null +++ b/packages/core/test/goalContainerSupervisor.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + DEFAULT_GOAL_CONTAINER_RETENTION, + buildGoalContainerLayout, +} from '../src/agents/goalSession/GoalContainerSupervisor.js'; +import { + GoalSessionContractError, + assertCredentialFreeRecoveryMetadata, +} from '../src/agents/goalSession/GoalSessionSupervisor.js'; + +const base = '/var/lib/propr/goal-runtime'; +const execution = { + goalId: 'goal-one', + sessionId: 'session-one', + controllerEpoch: 3, + turnId: 'turn-one', + executionId: 'execution-one', + attemptId: 'attempt-one', +}; + +test('container resources are stable within a goal session and isolated across goals', () => { + const first = buildGoalContainerLayout(base, execution); + const nextTurn = buildGoalContainerLayout(base, { ...execution, turnId: 'turn-two', attemptId: 'attempt-two' }); + const otherGoal = buildGoalContainerLayout(base, { ...execution, goalId: 'goal-two' }); + + assert.equal(first.providerHome, nextTurn.providerHome); + assert.equal(first.sessionRoot, nextTurn.sessionRoot); + assert.notEqual(first.executionId, nextTurn.executionId); + assert.notEqual(first.sessionRoot, otherGoal.sessionRoot); + assert.ok(first.providerHome.startsWith(`${base}/goals/`)); + assert.match(first.containerName, /^propr-goal-[a-f0-9-]+$/); +}); + +test('terminal container retention is explicit and keeps failures longer', () => { + assert.equal(DEFAULT_GOAL_CONTAINER_RETENTION.succeededMs, 24 * 60 * 60 * 1000); + assert.equal(DEFAULT_GOAL_CONTAINER_RETENTION.cancelledMs, 24 * 60 * 60 * 1000); + assert.equal(DEFAULT_GOAL_CONTAINER_RETENTION.failedMs, 7 * 24 * 60 * 60 * 1000); +}); + +test('credential-like recovery metadata is rejected before persistence', () => { + assert.doesNotThrow(() => assertCredentialFreeRecoveryMetadata({ checkpoint: 'cp-1', cursor: 5 })); + assert.throws( + () => assertCredentialFreeRecoveryMetadata({ checkpoint: 'cp-1', api_token: 'must-not-persist' }), + (error: unknown) => error instanceof GoalSessionContractError + && error.code === 'RECOVERY_METADATA_CONTAINS_CREDENTIAL', + ); +}); diff --git a/packages/core/test/goalSessionCapabilities.test.ts b/packages/core/test/goalSessionCapabilities.test.ts new file mode 100644 index 000000000..feaf1162b --- /dev/null +++ b/packages/core/test/goalSessionCapabilities.test.ts @@ -0,0 +1,899 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalCancelRequest, + GoalModelChangeRequest, + GoalProviderOpenRequest, + GoalPendingCancellationContext, + GoalProviderReconcileRequest, + GoalProviderReconcileResult, + GoalProviderSessionSnapshot, + GoalProviderTurnContext, + GoalSessionAdapter, + GoalSessionControlFence, + GoalSessionEvent, + GoalSessionIdentity, + GoalSessionState, + GoalTerminalCommit, +} from '../src/agents/goalSession/contract.js'; +import { + EAGER_ACTIVE_TURN_PROVIDER_CAPABILITIES, + FIRST_TURN_BOUNDARY_PROVIDER_CAPABILITIES, + GoalSessionContractError, + GoalSessionSupervisor, + StaleGoalSessionFenceError, +} from '../src/agents/goalSession/index.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; + +const identity = { goalId: 'goal-capabilities', sessionId: 'session-capabilities' }; +const repository = { + repository: 'integry/propr', + worktreePath: '/tmp/propr-goal-capabilities', + branch: 'goal-capability-branch', + headSha: 'abc123', +}; +const firstFence = { ...identity, controllerEpoch: 1, turnId: 'turn-one' }; + +class FirstTurnBoundaryAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'boundary-fake'; + readonly capabilities = FIRST_TURN_BOUNDARY_PROVIDER_CAPABILITIES; + openCalls = 0; + pauseCalls = 0; + steeringCalls = 0; + resumeTurnCalls = 0; + resumeSessionCalls = 0; + modelCalls: string[] = []; + actions: string[] = []; + contexts: GoalProviderTurnContext[] = []; + requests: GoalBeginTurnRequest[] = []; + turnStarted: (() => void) | undefined; + holdTurn: Promise | undefined; + emitIdentity = true; + acknowledgeMessages = true; + reconcileResult: GoalProviderReconcileResult = { outcome: 'failed', reason: 'not used by capability tests' }; + reconcileRequests: GoalProviderReconcileRequest[] = []; + pendingCancelContexts: GoalPendingCancellationContext[] = []; + pendingCancelStarted: (() => void) | undefined; + holdPendingCancel: Promise | undefined; + + async openSession(request: GoalProviderOpenRequest): Promise { + this.openCalls += 1; + if (!request.persisted) throw new Error('A bound first-turn session must resume from its persisted native ID'); + return request.persisted; + } + + async *beginTurn( + request: GoalBeginTurnRequest, + context: GoalProviderTurnContext, + ): AsyncIterable { + this.actions.push(`begin:${request.turnId}`); + this.requests.push(structuredClone(request)); + this.contexts.push(structuredClone(context)); + if (context.binding === 'pending' && this.emitIdentity) { + yield { + type: 'checkpoint', + checkpointId: 'first-init', + providerSessionId: 'native-first-turn-id', + recoveryMetadata: { conversation: 'native-first-turn-id' }, + }; + } + if (request.modelChange) { + yield { + type: 'model_changed', model: request.requestedModel, + providerEventId: `model-${request.modelChange.modelChangeId}-${request.modelChange.generation}`, + }; + } + this.turnStarted?.(); + if (this.holdTurn) await this.holdTurn; + if (this.acknowledgeMessages) { + for (const message of request.correctiveMessages ?? []) { + yield { type: 'message_acknowledged', messageId: message.messageId }; + } + } + yield { type: 'output', channel: 'stdout', data: `completed ${request.turnId}\n` }; + yield { type: 'completion', outcome: 'succeeded' }; + } + + async deliverMessage(): Promise<{ messageId: string }> { + this.steeringCalls += 1; + throw new Error('next-turn steering must not use an active-turn channel'); + } + + async requestPause(): Promise<{ appliesAt: 'after_turn' }> { + this.pauseCalls += 1; + throw new Error('after-turn pause must not interrupt the provider invocation'); + } + + async *resumeTurn(): AsyncIterable { + this.resumeTurnCalls += 1; + throw new Error('after-turn providers cannot resume the completed invocation'); + } + + async resumeSession( + _request: GoalSessionControlFence, + snapshot: GoalProviderSessionSnapshot, + ): Promise { + this.resumeSessionCalls += 1; + return snapshot; + } + + async requestModelChange( + request: GoalModelChangeRequest, + ): Promise<{ requestedModel: string; appliesAt: 'immediate'; effectiveModel: string }> { + this.actions.push(`model:${request.model}`); + this.modelCalls.push(request.model); + return { requestedModel: request.model, appliesAt: 'immediate', effectiveModel: request.model }; + } + + async cancel(_request: GoalCancelRequest): Promise {} + + async cancelPending( + _request: GoalCancelRequest, + pending: GoalPendingCancellationContext, + ): Promise { + this.pendingCancelContexts.push(structuredClone(pending)); + this.pendingCancelStarted?.(); + if (this.holdPendingCancel) await this.holdPendingCancel; + } + + async reconcile(request: GoalProviderReconcileRequest): Promise { + this.reconcileRequests.push(structuredClone(request)); + return this.reconcileResult; + } +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +class GatedCompletionLoadPorts extends InMemoryGoalSessionPorts { + private nextRunningLoad: { + loaded: ReturnType; + release: ReturnType; + } | undefined; + + gateNextRunningLoad(): { loaded: Promise; release: () => void } { + const gate = { loaded: deferred(), release: deferred() }; + this.nextRunningLoad = gate; + return { loaded: gate.loaded.promise, release: gate.release.resolve }; + } + + override async load(request: GoalSessionIdentity): Promise { + const state = await super.load(request); + const gate = this.nextRunningLoad; + if (gate && state?.status === 'running') { + this.nextRunningLoad = undefined; + gate.loaded.resolve(); + await gate.release.promise; + } + return state; + } +} + +class GatedLazyOpenPorts extends InMemoryGoalSessionPorts { + private openGate: { blocked: ReturnType; release: ReturnType } | undefined; + + gateNextLazyOpen(): { blocked: Promise; release: () => void } { + const gate = { blocked: deferred(), release: deferred() }; + this.openGate = gate; + return { blocked: gate.blocked.promise, release: gate.release.resolve }; + } + + override async compareAndSet( + expected: GoalSessionState, + next: Omit, + ): Promise { + const gate = this.openGate; + if (gate && expected.status === 'initializing' && next.status === 'idle' && !expected.providerSessionId) { + this.openGate = undefined; + gate.blocked.resolve(); + await gate.release.promise; + } + return super.compareAndSet(expected, next); + } +} + +class ContractIdempotencyPorts extends InMemoryGoalSessionPorts { + readonly terminalCommitKeys: string[] = []; + + override async commit( + expected: GoalSessionState, + next: Omit, + completion: GoalTerminalCommit, + ): Promise { + this.terminalCommitKeys.push(JSON.stringify([ + completion.scope, + completion.fence.goalId, + completion.fence.sessionId, + completion.fence.controllerEpoch, + completion.scope === 'turn' ? completion.fence.turnId : null, + completion.execution.executionId, + completion.execution.attemptId, + ])); + return super.commit(expected, next, completion); + } +} + +test('capability fixtures describe eager-active and lazy-boundary providers without overlap', () => { + assert.deepEqual(EAGER_ACTIVE_TURN_PROVIDER_CAPABILITIES, { + nativeSessionId: 'eager', + steering: 'active_turn', + pause: 'active_turn', + modelChange: 'next_safe_boundary', + }); + assert.deepEqual(FIRST_TURN_BOUNDARY_PROVIDER_CAPABILITIES, { + nativeSessionId: 'first_turn', + firstTurnIdCrashPolicy: 'fail', + steering: 'next_turn', + pause: 'after_turn', + modelChange: 'next_turn', + }); +}); + +test('lazy-ID cancellation while open reaches one durable terminal outcome without a native ID', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const opened = await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + assert.equal(opened.providerSessionId, undefined); + + const cancelled = await supervisor.cancel({ ...identity, controllerEpoch: 1, reason: 'cancel before first turn' }); + assert.equal(cancelled.status, 'terminated'); + assert.equal(cancelled.activeTurn, undefined); + assert.equal(cancelled.initializationIntent, undefined); + assert.equal(adapter.pendingCancelContexts.length, 1); + assert.equal(adapter.pendingCancelContexts[0]?.activeTurn, undefined); + assert.equal(adapter.pendingCancelContexts[0]?.initializationIntent.deterministicOpenKey, + opened.initializationIntent?.deterministicOpenKey); + + const repeated = await supervisor.cancel({ ...identity, controllerEpoch: 1, reason: 'repeat cancellation' }); + assert.equal(repeated.status, 'terminated'); + assert.equal(adapter.pendingCancelContexts.length, 1, 'repeat cancel does not signal a terminal session again'); + const completions = (await persistence.replay(identity)).filter(record => record.event.type === 'completion'); + assert.equal(completions.length, 1); + assert.equal(completions[0]?.event.type === 'completion' ? completions[0].event.outcome : '', 'cancelled'); +}); + +test('lazy-ID cancellation wins while open is persisting its pending boundary', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new GatedLazyOpenPorts(); + const gate = persistence.gateNextLazyOpen(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const opening = supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + await gate.blocked; + + const terminal = await supervisor.cancel({ ...identity, controllerEpoch: 1, reason: 'cancel during lazy open' }); + assert.equal(terminal.status, 'terminated'); + assert.equal(terminal.activeTurn, undefined); + assert.equal(adapter.pendingCancelContexts.length, 1); + gate.release(); + await assert.rejects(opening, StaleGoalSessionFenceError); + assert.equal((await persistence.load(identity))?.status, 'terminated'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('lazy-ID cancellation fences provider completion before the first checkpoint', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + adapter.emitIdentity = false; + const turnStarted = deferred(); + const releaseTurn = deferred(); + const cancelStarted = deferred(); + const releaseCancel = deferred(); + adapter.turnStarted = turnStarted.resolve; + adapter.holdTurn = releaseTurn.promise; + adapter.pendingCancelStarted = cancelStarted.resolve; + adapter.holdPendingCancel = releaseCancel.promise; + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const running = supervisor.runTurn({ + ...firstFence, + executionId: 'execution-cancel-before-id', + attemptId: 'attempt-cancel-before-id', + objective: 'cancel the provider before it binds a native ID', + repository, + requestedModel: 'model-a', + }); + await turnStarted.promise; + + const cancelling = supervisor.cancel({ ...identity, controllerEpoch: 1, reason: 'cancel pending provider' }); + await cancelStarted.promise; + assert.equal((await persistence.load(identity))?.status, 'cancelling'); + assert.deepEqual(adapter.pendingCancelContexts[0]?.activeTurn, { + turnId: firstFence.turnId, + executionId: 'execution-cancel-before-id', + attemptId: 'attempt-cancel-before-id', + }); + releaseTurn.resolve(); + await assert.rejects(running); + assert.equal((await persistence.load(identity))?.status, 'cancelling'); + + releaseCancel.resolve(); + const terminal = await cancelling; + assert.equal(terminal.status, 'terminated'); + assert.equal(terminal.activeTurn, undefined); + assert.equal(terminal.providerSessionId, undefined); + const completions = (await persistence.replay(identity)).filter(record => record.event.type === 'completion'); + assert.equal(completions.length, 1); + assert.equal(completions[0]?.event.type === 'completion' ? completions[0].event.outcome : '', 'cancelled'); +}); + +test('a restarted lazy-ID controller finishes a cancellation claimed before a crash', async () => { + const firstAdapter = new FirstTurnBoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const initial = new GoalSessionSupervisor(firstAdapter, persistence.asRuntimePorts()); + const opened = await initial.openSession({ ...identity, provider: firstAdapter.provider, controllerEpoch: 1 }); + const { version: _version, ...withoutVersion } = opened; + const claimed = await persistence.compareAndSet(opened, { + ...withoutVersion, + status: 'cancelling', + providerOperationGeneration: 1, + cancellationIntent: { + cancellationId: 'crashed-cancellation', reason: 'resume cancellation after crash', + claimedAt: new Date().toISOString(), + pendingContext: { initializationIntent: opened.initializationIntent! }, + }, + providerBarrierIntent: { + generation: 1, operationId: 'crashed-cancellation', kind: 'cancellation', phase: 'published', + claimedAt: new Date().toISOString(), pendingCancellationId: 'crashed-cancellation', + }, + updatedAt: new Date().toISOString(), + }); + assert.equal(claimed?.status, 'cancelling'); + + const replacementAdapter = new FirstTurnBoundaryAdapter(); + const replacement = new GoalSessionSupervisor(replacementAdapter, persistence.asRuntimePorts()); + const terminal = await replacement.cancel({ ...identity, controllerEpoch: 1, reason: 'resume cancellation after crash' }); + assert.equal(terminal.status, 'terminated'); + assert.equal(terminal.activeTurn, undefined); + assert.equal(replacementAdapter.pendingCancelContexts.length, 1); + assert.equal((await replacement.cancel({ ...identity, controllerEpoch: 1, reason: 'idempotent retry' })).status, 'terminated'); + assert.equal(replacementAdapter.pendingCancelContexts.length, 1); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('first-turn identity, FIFO next-turn ack, and after-turn pause/resume stay boundary-safe', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const opened = await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + assert.equal(opened.status, 'idle'); + assert.equal(opened.providerSessionId, undefined, 'no placeholder provider ID is invented'); + assert.ok(opened.initializationIntent?.deterministicOpenKey); + assert.equal(adapter.openCalls, 0, 'the native ID is not eagerly opened'); + + persistence.enqueueMessage({ ...identity, messageId: 'message-one', body: 'first correction' }); + persistence.enqueueMessage({ ...identity, messageId: 'message-two', body: 'second correction' }); + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + const running = supervisor.runTurn({ + ...firstFence, + executionId: 'execution-one', + attemptId: 'attempt-one', + objective: 'Run a discrete provider turn', + repository, + requestedModel: 'model-a', + }); + await started; + + const steering = await supervisor.deliverMessage({ ...firstFence, messageId: 'message-one', body: 'ignored copy' }); + assert.deepEqual(steering, { + outcome: 'unsupported_same_turn', messageId: 'message-one', supportedBoundary: 'next_turn', + }); + const pause = await supervisor.requestPause({ ...firstFence, reason: 'pause after this invocation' }); + assert.deepEqual(pause, { appliesAt: 'after_turn' }); + assert.equal(adapter.pauseCalls, 0, 'pause never maps to an interrupt/terminal provider call'); + releaseTurn(); + + const finished = await running; + assert.equal(finished.state.status, 'paused'); + assert.equal(finished.state.activeTurn, undefined, 'there is no active provider turn at an after-turn pause'); + assert.equal(finished.state.providerSessionId, 'native-first-turn-id'); + assert.equal(finished.state.initializationIntent, undefined); + assert.equal(adapter.contexts[0]?.binding, 'pending'); + assert.deepEqual(adapter.requests[0]?.correctiveMessages?.map(message => message.messageId), ['message-one', 'message-two']); + assert.deepEqual((await persistence.listPending(identity)).map(message => message.messageId), []); + assert.equal(adapter.steeringCalls, 0); + + assert.deepEqual(await supervisor.resumeTurn(firstFence), { + disposition: 'unsupported_same_turn', supportedBoundary: 'after_turn', + }); + assert.equal(adapter.resumeTurnCalls, 0); + await assert.rejects( + supervisor.runTurn({ + ...firstFence, turnId: 'turn-two', executionId: 'execution-two', objective: 'must stay paused', + repository, requestedModel: 'model-a', + }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'SESSION_NOT_IDLE', + ); + + await supervisor.requestModelChange({ ...identity, controllerEpoch: 1, model: 'model-b' }); + assert.deepEqual(adapter.modelCalls, [], 'next-turn model changes are not applied mid-boundary'); + const resumed = await supervisor.resumeSession({ ...identity, controllerEpoch: 1 }); + assert.equal(resumed.status, 'idle'); + assert.equal(resumed.providerSessionId, 'native-first-turn-id'); + + adapter.holdTurn = undefined; + adapter.turnStarted = undefined; + const second = await supervisor.runTurn({ + ...firstFence, + turnId: 'turn-two', + executionId: 'execution-two', + objective: 'Start only after boundary resume', + repository, + requestedModel: 'model-b', + }); + assert.equal(second.state.status, 'idle'); + assert.deepEqual(adapter.modelCalls, []); + assert.deepEqual(adapter.actions.slice(-1), ['begin:turn-two']); + assert.equal(adapter.contexts[1]?.binding, 'bound'); + assert.equal(adapter.contexts[1]?.binding === 'bound' + ? adapter.contexts[1].snapshot.providerSessionId + : undefined, 'native-first-turn-id'); + + const acknowledgedIds = (await persistence.replay(identity)) + .filter(record => record.event.type === 'message_acknowledged') + .map(record => record.event.type === 'message_acknowledged' ? record.event.messageId : ''); + assert.deepEqual(acknowledgedIds, ['message-one', 'message-two']); +}); + +test('after-turn completion honors a pause acknowledged after its pre-completion state read', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new GatedCompletionLoadPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + + const providerRelease = deferred(); + const providerStarted = deferred(); + adapter.holdTurn = providerRelease.promise; + adapter.turnStarted = providerStarted.resolve; + const running = supervisor.runTurn({ + ...firstFence, + executionId: 'execution-pause-race', + attemptId: 'attempt-pause-race', + objective: 'complete concurrently with an after-turn pause', + repository, + requestedModel: 'model-a', + }); + await providerStarted.promise; + + const completionLoad = persistence.gateNextRunningLoad(); + providerRelease.resolve(); + await completionLoad.loaded; + assert.deepEqual(await supervisor.requestPause({ ...firstFence, reason: 'pause at completion' }), { + appliesAt: 'after_turn', + }); + assert.equal((await persistence.load(identity))?.status, 'pause_requested'); + completionLoad.release(); + + const finished = await running; + assert.equal(finished.state.status, 'paused'); + assert.equal(finished.state.activeTurn, undefined); + assert.equal((await persistence.load(identity))?.status, 'paused'); + const terminalEvents = (await persistence.replay(identity)).filter(record => + record.turnId === firstFence.turnId + && (record.event.type === 'pause_boundary' || record.event.type === 'completion')); + assert.deepEqual(terminalEvents.map(record => record.event.type), ['pause_boundary', 'completion']); + assert.equal(terminalEvents[0]?.event.type === 'pause_boundary' + ? terminalEvents[0].event.boundary : '', 'after_turn'); +}); + +test('late after-turn pause state and canonical audit boundary survive an ambiguous commit crash exactly once', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new GatedCompletionLoadPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const providerRelease = deferred(); + const providerStarted = deferred(); + adapter.holdTurn = providerRelease.promise; + adapter.turnStarted = providerStarted.resolve; + const request = { + ...firstFence, + executionId: 'execution-pause-crash', + attemptId: 'attempt-pause-crash', + objective: 'commit late pause and completion atomically', + repository, + requestedModel: 'model-a', + }; + const running = supervisor.runTurn(request); + await providerStarted.promise; + + const completionLoad = persistence.gateNextRunningLoad(); + providerRelease.resolve(); + await completionLoad.loaded; + await supervisor.requestPause({ ...firstFence, reason: 'pause in terminal crash window' }); + persistence.setTerminalFault('after_commit'); + completionLoad.release(); + await assert.rejects(running, /Injected crash after terminal transaction commit/); + + const saved = await persistence.load(identity); + assert.equal(saved?.status, 'paused'); + assert.equal(saved?.activeTurn, undefined); + const terminalEvents = (await persistence.replay(identity)).filter(record => + record.turnId === firstFence.turnId + && (record.event.type === 'pause_boundary' || record.event.type === 'completion')); + assert.deepEqual(terminalEvents.map(record => record.event.type), ['pause_boundary', 'completion']); + + const restarted = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + assert.equal((await restarted.runTurn(request)).disposition, 'duplicate'); + const replayed = (await persistence.replay(identity)).filter(record => + record.turnId === firstFence.turnId + && (record.event.type === 'pause_boundary' || record.event.type === 'completion')); + assert.deepEqual(replayed.map(record => record.event.type), ['pause_boundary', 'completion']); +}); + +test('late after-turn pause boundary replays atomically after a pre-commit crash', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new GatedCompletionLoadPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const providerRelease = deferred(); + const providerStarted = deferred(); + adapter.holdTurn = providerRelease.promise; + adapter.turnStarted = providerStarted.resolve; + const request = { + ...firstFence, + executionId: 'execution-pause-precommit', + attemptId: 'attempt-pause-precommit', + objective: 'recover the atomic late-pause terminal transaction', + repository, + requestedModel: 'model-a', + }; + const running = supervisor.runTurn(request); + await providerStarted.promise; + await supervisor.requestPause({ ...firstFence, reason: 'pause before terminal commit' }); + persistence.setTerminalFault('before_commit_always'); + providerRelease.resolve(); + await assert.rejects(running, /Injected crash before terminal transaction commit/); + + const preCommit = await persistence.load(identity); + assert.equal(preCommit?.status, 'pause_requested'); + assert.equal(preCommit?.pendingAfterTurnPause, true); + assert.deepEqual((await persistence.replay(identity)).filter(record => + record.turnId === firstFence.turnId + && (record.event.type === 'pause_boundary' || record.event.type === 'completion')), []); + + persistence.setTerminalFault(undefined); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'worker crashed before terminal commit' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { + outcome: 'resumed', + snapshot: { + providerSessionId: 'native-first-turn-id', + recoveryMetadata: { conversation: 'native-first-turn-id', checkpoint: 'terminal-retry' }, + model: 'model-a', + }, + reason: 'retry the discrete invocation from its durable checkpoint', + }; + adapter.holdTurn = undefined; + adapter.turnStarted = undefined; + const restarted = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const reconciled = await restarted.reconcile(identity, 2, repository); + assert.equal(reconciled.state.status, 'paused'); + assert.equal(reconciled.state.pendingAfterTurnPause, true); + const recovered = await restarted.resumeTurn({ ...identity, controllerEpoch: 2 }); + assert.equal(recovered.disposition, 'started'); + assert.equal(recovered.state.status, 'paused'); + assert.equal(recovered.state.activeTurn, undefined); + assert.equal(recovered.state.pendingAfterTurnPause, undefined); + const replayed = (await persistence.replay(identity)).filter(record => + record.turnId === firstFence.turnId + && (record.event.type === 'pause_boundary' || record.event.type === 'completion')); + assert.deepEqual(replayed.map(record => record.event.type), ['pause_boundary', 'completion']); + assert.equal(replayed[0]?.event.type === 'pause_boundary' ? replayed[0].event.boundary : '', 'after_turn'); + assert.equal(replayed[0]?.attemptId, recovered.execution.attemptId); + assert.equal(replayed[1]?.attemptId, recovered.execution.attemptId); +}); + +test('first-turn providers reject authoritative output before a real native ID is bound', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + adapter.emitIdentity = false; + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + + await assert.rejects( + supervisor.runTurn({ + ...firstFence, + executionId: 'execution-unbound', + objective: 'Never accept fake identity output', + repository, + requestedModel: 'model-a', + }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'FIRST_TURN_ID_NOT_BOUND', + ); + const replay = await persistence.replay(identity); + assert.equal(replay.some(record => record.event.type === 'output'), false); + assert.equal((await persistence.load(identity))?.providerSessionId, undefined); +}); + +test('reopen cleans an already-terminal first-turn failure without a new epoch completion', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + adapter.emitIdentity = false; + const persistence = new ContractIdempotencyPorts(); + const initial = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await initial.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + + await assert.rejects(initial.runTurn({ + ...firstFence, + executionId: 'execution-terminal-failure', + attemptId: 'attempt-terminal-failure', + objective: 'fail ordinarily before native identity binding', + repository, + requestedModel: 'model-a', + }), (error: unknown) => error instanceof GoalSessionContractError && error.code === 'FIRST_TURN_ID_NOT_BOUND'); + const terminal = await persistence.load(identity); + assert.equal(terminal?.status, 'failed'); + assert.equal(terminal?.activeTurn?.status, 'failed'); + assert.equal(persistence.terminalCommitKeys.length, 1); + + const replacement = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await assert.rejects( + replacement.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'FIRST_TURN_ID_NOT_BOUND', + ); + + const cleaned = await persistence.load(identity); + assert.equal(cleaned?.controllerEpoch, 1, 'terminal cleanup cannot transfer controller ownership'); + assert.equal(cleaned?.status, 'failed'); + assert.equal(cleaned?.activeTurn, undefined); + assert.equal(cleaned?.initializationIntent, undefined); + assert.equal(persistence.terminalCommitKeys.length, 1, 'the exact contract key is never retried under epoch two'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('fail policy durably fails and clears an unbound first turn after crash and reopen', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const initial = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'initialization-attempt'); + const opened = await initial.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const { version: _version, ...persisted } = opened; + const crashed = await persistence.compareAndSet(opened, { + ...persisted, + status: 'running', + activeTurn: { + turnId: firstFence.turnId, + executionId: 'execution-crashed', + attemptId: 'attempt-crashed', + executionEpoch: 1, + objective: 'crash before native identity checkpoint', + requestedModel: 'model-a', + repository, + status: 'running', + }, + }); + assert.ok(crashed); + + const replacement = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await assert.rejects( + replacement.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'FIRST_TURN_ID_NOT_BOUND', + ); + const failed = await persistence.load(identity); + assert.equal(failed?.status, 'failed'); + assert.equal(failed?.activeTurn, undefined); + assert.equal(failed?.initializationIntent, undefined); + assert.match(failed?.failureReason ?? '', /before binding its native session ID/); + const completions = (await persistence.replay(identity)).filter(record => record.event.type === 'completion'); + assert.equal(completions.length, 1); + assert.equal(completions[0]?.event.type === 'completion' ? completions[0].event.outcome : '', 'failed'); + + const terminalVersion = failed?.version; + await assert.rejects( + replacement.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'FIRST_TURN_ID_NOT_BOUND', + ); + assert.equal((await persistence.load(identity))?.version, terminalVersion, 'repeated opens do not mutate terminal state'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('first-turn binding consumes its deferred requested model without reapplying it on turn two', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + await supervisor.requestModelChange({ ...identity, controllerEpoch: 1, model: 'model-b' }); + + const first = await supervisor.runTurn({ + ...firstFence, + executionId: 'execution-model-one', + objective: 'apply deferred model in the first invocation', + repository, + requestedModel: 'model-a', + }); + assert.equal(first.state.currentModel, 'model-b'); + assert.equal(first.state.pendingModelChange, undefined); + assert.deepEqual(adapter.modelCalls, [], 'the first invocation receives its model directly'); + + await supervisor.runTurn({ + ...firstFence, + turnId: 'turn-two', + executionId: 'execution-model-two', + objective: 'do not redundantly reapply the first-turn model', + repository, + requestedModel: 'model-b', + }); + assert.deepEqual(adapter.modelCalls, []); + assert.deepEqual(adapter.requests.map(request => request.requestedModel), ['model-b', 'model-b']); +}); + +test('successful completion without acknowledging supplied messages is a protocol violation', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + adapter.acknowledgeMessages = false; + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + persistence.enqueueMessage({ ...identity, messageId: 'message-unacknowledged', body: 'must be accepted' }); + + await assert.rejects( + supervisor.runTurn({ + ...firstFence, + executionId: 'execution-unacknowledged', + objective: 'provider must acknowledge supplied messages', + repository, + requestedModel: 'model-a', + }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'MESSAGE_ACK_MISSING', + ); + assert.equal((await persistence.load(identity))?.status, 'failed'); + assert.deepEqual((await persistence.listPending(identity)).map(message => message.messageId), ['message-unacknowledged']); + const completion = (await persistence.replay(identity)).find(record => record.event.type === 'completion'); + assert.equal(completion?.event.type === 'completion' ? completion.event.outcome : '', 'failed'); +}); + +test('deterministic first-turn retry mints a fresh attempt instead of reusing the crashed invocation', async () => { + class RetryingFirstTurnAdapter extends FirstTurnBoundaryAdapter { + override readonly capabilities = { + ...FIRST_TURN_BOUNDARY_PROVIDER_CAPABILITIES, + firstTurnIdCrashPolicy: 'retry_deterministically', + } as const; + } + const adapter = new RetryingFirstTurnAdapter(); + adapter.emitIdentity = false; + const persistence = new InMemoryGoalSessionPorts(); + const initial = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'initialization-attempt'); + await initial.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + await assert.rejects(initial.runTurn({ + ...firstFence, + executionId: 'execution-retry', + attemptId: 'crashed-attempt', + objective: 'first invocation crashes', + repository, + requestedModel: 'model-a', + }), /native session ID/); + + const ids = ['recovered-initialization-attempt', 'fresh-provider-attempt']; + const replacement = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => ids.shift()!); + const reopened = await replacement.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }); + assert.equal(reopened.status, 'idle'); + assert.equal(reopened.activeTurn, undefined); + assert.equal(reopened.retryTurn?.crashedAttemptId, 'crashed-attempt'); + assert.equal(reopened.initializationIntent?.attemptId, 'recovered-initialization-attempt'); + adapter.emitIdentity = true; + const recovered = await replacement.runTurn({ + ...firstFence, + controllerEpoch: 2, + executionId: 'execution-retry', + attemptId: 'crashed-attempt', + objective: 'retry the same logical turn', + repository, + requestedModel: 'model-a', + }); + + assert.equal(recovered.execution.executionId, 'execution-retry'); + assert.equal(recovered.execution.attemptId, 'fresh-provider-attempt'); + assert.notEqual(recovered.execution.attemptId, 'crashed-attempt'); + assert.equal(adapter.requests.at(-1)?.attemptId, 'fresh-provider-attempt'); +}); + +test('first-turn after-turn profile reconciles a post-ID container loss through a fresh fenced invocation', async () => { + const adapter = new FirstTurnBoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const initial = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await initial.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const bound = await initial.runTurn({ + ...firstFence, + executionId: 'execution-binding-turn', + attemptId: 'attempt-binding-turn', + objective: 'bind the native session before the later crash', + repository, + requestedModel: 'model-a', + }); + const { version: _version, ...persisted } = bound.state; + const crashed = await persistence.compareAndSet(bound.state, { + ...persisted, + status: 'running', + activeTurn: { + turnId: 'turn-crashed-after-binding', + executionId: 'execution-crashed-after-binding', + attemptId: 'attempt-crashed-after-binding', + executionEpoch: 1, + objective: 'continue the bound session after container loss', + requestedModel: 'model-a', + repository, + status: 'running', + }, + }); + assert.ok(crashed); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'container was lost' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { + outcome: 'resumed', + snapshot: { + providerSessionId: 'native-first-turn-id', + recoveryMetadata: { conversation: 'native-first-turn-id', checkpoint: 'reconciled' }, + model: 'model-a', + }, + reason: 'the bound provider session is recoverable', + }; + const attemptIds = ['attempt-reconciliation', 'attempt-continuation']; + const replacement = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => attemptIds.shift()!); + + const reconciled = await replacement.reconcile(identity, 2, repository); + assert.equal(reconciled.state.status, 'paused'); + assert.equal(reconciled.state.activeTurn?.status, 'paused'); + assert.equal(reconciled.state.activeTurn?.attemptId, 'attempt-reconciliation'); + assert.equal(adapter.reconcileRequests[0]?.attemptId, 'attempt-reconciliation'); + + await replacement.requestModelChange({ ...identity, controllerEpoch: 2, model: 'model-recovered' }); + assert.equal((await persistence.load(identity))?.pendingModelChange, 'model-recovered'); + + const continuationStarted = deferred(); + const continuationRelease = deferred(); + adapter.turnStarted = continuationStarted.resolve; + adapter.holdTurn = continuationRelease.promise; + const continuation = replacement.resumeTurn({ ...identity, controllerEpoch: 2 }); + await continuationStarted.promise; + assert.equal((await persistence.load(identity))?.activeTurn?.attemptId, 'attempt-continuation'); + const staleAppend = await persistence.append( + { ...identity, controllerEpoch: 2, turnId: 'turn-crashed-after-binding' }, + { executionId: 'execution-crashed-after-binding', attemptId: 'attempt-reconciliation' }, + { type: 'output', channel: 'stdout', data: 'late output from reconciliation attempt' }, + ); + assert.deepEqual(staleAppend, { accepted: false, reason: 'turn_not_active' }); + continuationRelease.resolve(); + const continued = await continuation; + assert.equal(continued.disposition, 'started'); + assert.equal(continued.state.status, 'idle'); + assert.equal(continued.execution.executionId, 'execution-crashed-after-binding'); + assert.equal(continued.execution.attemptId, 'attempt-continuation'); + assert.notEqual(continued.execution.attemptId, adapter.reconcileRequests[0]?.attemptId); + assert.equal(adapter.resumeTurnCalls, 0, 'crash retry uses a fresh discrete invocation, not operator same-turn resume'); + assert.equal(adapter.requests.at(-1)?.turnId, 'turn-crashed-after-binding'); + assert.equal(adapter.requests.at(-1)?.attemptId, 'attempt-continuation'); + assert.equal(adapter.requests.at(-1)?.requestedModel, 'model-recovered'); + assert.equal(adapter.actions.at(-1), 'begin:turn-crashed-after-binding'); + assert.equal(adapter.requests.at(-1)?.modelChange?.modelChangeId, + (await persistence.load(identity))?.modelChangeIntents?.find(intent => intent.model === 'model-recovered')?.modelChangeId); + assert.deepEqual(adapter.modelCalls, [], 'next-turn intent is supplied at the actual invocation, not through a side call'); + assert.equal(adapter.contexts.at(-1)?.binding, 'bound'); + assert.equal((await persistence.load(identity))?.currentModel, 'model-recovered'); + assert.equal((await persistence.load(identity))?.pendingModelChange, undefined); + + const recoveredCompletions = (await persistence.replay(identity)).filter(record => + record.turnId === 'turn-crashed-after-binding' && record.event.type === 'completion'); + assert.equal(recoveredCompletions.length, 1); + assert.equal(recoveredCompletions[0]?.attemptId, 'attempt-continuation'); + const recoveredModelAcknowledgements = (await persistence.replay(identity)).filter(record => + record.event.type === 'model_change_acknowledged' + && record.event.requestedModel === 'model-recovered'); + assert.equal(recoveredModelAcknowledgements.length, 1); +}); diff --git a/packages/core/test/goalSessionExactHeadCorrection.test.ts b/packages/core/test/goalSessionExactHeadCorrection.test.ts new file mode 100644 index 000000000..c85270727 --- /dev/null +++ b/packages/core/test/goalSessionExactHeadCorrection.test.ts @@ -0,0 +1,721 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import Database from 'better-sqlite3'; +import type { + GoalBeginTurnRequest, GoalProviderCancelRequest, GoalProviderOpenContext, GoalProviderOpenRequest, + GoalModelChangeRequest, GoalProviderSessionSnapshot, GoalSessionAdapter, GoalSessionEvent, GoalSessionState, +} from '../src/agents/goalSession/contract.js'; +import { openSupervisedCodexAppServer } from '../src/agents/goalSession/CodexAppServerOpen.js'; +import { decodeDurableGoalSessionState } from '../src/agents/goalSession/durableStateSecurity.js'; +import { GoalSessionSupervisor } from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { GoalSessionContractError } from '../src/agents/goalSession/errors.js'; +import { issueGoalSupervisedOpenPlan } from '../src/agents/goalSession/goalSessionOpen.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { createSqliteGoalSessionRuntimePorts } from '../src/agents/goalSession/SqliteGoalSessionControlDomain.js'; +import { sanitizeNewRecoveryMetadata, sanitizeRecoveryMetadata } from '../src/agents/goalSession/recoveryMetadata.js'; +import { + rebuildIteratorResult, rebuildMessageAcknowledgement, rebuildModelAcknowledgement, + rebuildPauseAcknowledgement, rebuildProviderSnapshot, rebuildReconcileResult, + untrustedProviderResult, +} from '../src/agents/goalSession/providerResultBoundary.js'; +import { createProductionSchema, recovery, seedAuthoritativeGoal } from './productionGoalSessionTestSupport.js'; + +const identity = { goalId: 'exact-correction-goal', sessionId: 'exact-correction-session' }; +const repository = { repository: 'integry/propr', worktreePath: '/tmp/exact-correction', branch: 'correction' }; + +function durableState(): GoalSessionState { + const timestamp = new Date().toISOString(); + return { + ...identity, provider: 'codec-adapter', providerSessionId: 'native-session', recoveryMetadata: {}, + controllerEpoch: 1, status: 'idle', currentModel: 'model-a', completedTurnIds: [], + version: 1, createdAt: timestamp, updatedAt: timestamp, + }; +} + +test('strict durable decoding rejects every malformed known field, accessors, and ambiguous cancellation identity', () => { + const base = durableState(); + const poisoned: unknown[] = [ + { ...base, goalId: 7 }, { ...base, sessionId: '' }, { ...base, provider: {} }, + { ...base, providerSessionId: 9 }, { ...base, recoveryMetadata: { command: 'provider --auth' } }, + { ...base, controllerEpoch: '1' }, { ...base, status: 'unknown' }, { ...base, currentModel: false }, + { ...base, requestedModel: [] }, { ...base, pendingModelChange: 1 }, { ...base, pendingAfterTurnPause: 'yes' }, + { ...base, completedTurnIds: ['turn-a', 'turn-a'] }, { ...base, version: 0 }, + { ...base, createdAt: 'yesterday' }, { ...base, failureReason: 'Bearer durable-secret' }, + { ...base, status: 'cancelling' }, + { ...base, providerOperationGeneration: 2, providerBarrierIntent: { + generation: 2, operationId: 'cancel-2', kind: 'cancellation', phase: 'pending', + claimedAt: base.createdAt, pendingCancellationId: 'different-cancel', + }, cancellationIntent: { cancellationId: 'cancel-2', reason: 'cancel', claimedAt: base.createdAt } }, + { ...base, usageAccounting: { version: 1, lastWatermark: 2, occurrences: ['usage-a', 'usage-a'] } }, + { ...base, modelChangeIntents: [ + { modelChangeId: 'model-2', model: 'b', requestedAt: base.createdAt, generation: 2 }, + { modelChangeId: 'model-1', model: 'a', requestedAt: base.createdAt, generation: 1 }, + ] }, + { ...base, status: 'running' }, + { ...base, status: 'idle', activeTurn: { + turnId: 'turn-live', executionId: 'execution-live', attemptId: 'attempt-live', executionEpoch: 1, + objective: 'objective', requestedModel: 'model-a', repository, status: 'running', + } }, + { ...base, status: 'terminated', activeTurn: { + turnId: 'turn-live', executionId: 'execution-live', attemptId: 'attempt-live', executionEpoch: 1, + objective: 'objective', requestedModel: 'model-a', repository, status: 'running', + } }, + { ...base, recoveryAttemptId: 'recovery-attempt', providerOperationGeneration: 2, + recoveryAttempt: { + operationToken: 'recovery-token', operationGeneration: 2, executionId: 'execution-live', + attemptId: 'recovery-attempt', controllerEpoch: 1, sessionStatus: 'idle', + claimedAt: base.createdAt, leaseExpiresAt: base.createdAt, phase: 'claimed', + }, + resumeIntent: { + executionId: 'execution-live', attemptId: 'resume-attempt', operationId: 'resume-id', + operationGeneration: 2, kind: 'after_turn', controllerEpoch: 1, + claimedAt: base.createdAt, leaseExpiresAt: base.createdAt, phase: 'claimed', + } }, + { ...base, providerOperationGeneration: 2, providerBarrierIntent: { + generation: 2, operationId: 'orphan-operation:lease-expiry', kind: 'lease_expiry', + phase: 'pending', claimedAt: base.createdAt, + } }, + { ...base, modelChangeGeneration: 2, modelChangeIntents: [ + { modelChangeId: 'duplicate', model: 'a', requestedAt: base.createdAt, generation: 1 }, + { modelChangeId: 'duplicate', model: 'b', requestedAt: base.createdAt, generation: 2 }, + ], modelChangeIntent: { modelChangeId: 'duplicate', model: 'b', requestedAt: base.createdAt, generation: 2 } }, + { ...base, modelChangeGeneration: 1, modelChangeIntents: [ + { modelChangeId: 'model-one', model: 'b', requestedAt: base.createdAt, generation: 1 }, + ], modelChangeIntent: { modelChangeId: 'different-tail', model: 'b', requestedAt: base.createdAt, generation: 1 } }, + { ...base, modelChangeGeneration: 1, modelChangeIntents: [{ + modelChangeId: 'model-one', model: 'b', requestedAt: base.createdAt, generation: 1, + phase: 'provider_in_doubt', applicationToken: 'token-one', + }], modelChangeIntent: { + modelChangeId: 'model-one', model: 'b', requestedAt: base.createdAt, generation: 1, + phase: 'provider_in_doubt', applicationToken: 'token-one', + } }, + { ...base, modelChangeGeneration: 1, modelChangeIntents: [{ + modelChangeId: 'model-one', model: 'b', requestedAt: base.createdAt, generation: 1, + phase: 'committed', acknowledgement: { + requestedModel: 'different-model', appliesAt: 'next_turn', effectiveModel: 'b', + }, + }], modelChangeIntent: { + modelChangeId: 'model-one', model: 'b', requestedAt: base.createdAt, generation: 1, + phase: 'committed', acknowledgement: { + requestedModel: 'different-model', appliesAt: 'next_turn', effectiveModel: 'b', + }, + } }, + { ...base, completedTurnIds: ['turn-done'], completedTurns: [{ + turnId: 'turn-done', executionId: 'execution-old', attemptId: 'attempt-old', + }], activeTurn: { + turnId: 'turn-done', executionId: 'execution-new', attemptId: 'attempt-new', executionEpoch: 1, + objective: 'objective', requestedModel: 'model-a', repository, status: 'completed', + } }, + { ...base, activeTurn: { + turnId: 'turn-a', executionId: 'execution-a', attemptId: 3, executionEpoch: 1, + objective: 'objective', requestedModel: 'model-a', repository, status: 'running', + } }, + { ...base, excessRuntimeField: 'must-not-cross' }, + ]; + for (const value of poisoned) assert.throws(() => decodeDurableGoalSessionState(value)); + + let getterRead = false; + const accessor = { ...base } as Record; + Object.defineProperty(accessor, 'goalId', { + enumerable: true, + get() { getterRead = true; return identity.goalId; }, + }); + assert.throws(() => decodeDurableGoalSessionState(accessor)); + assert.equal(getterRead, false, 'decoder rejects accessor fields without evaluating them'); + + const intent = { + modelChangeId: 'cross-attempt-model', model: 'model-b', requestedAt: base.createdAt, + generation: 1, phase: 'committed' as const, + acknowledgement: { + outcome: 'acknowledged' as const, requestedModel: 'model-b', + appliesAt: 'next_turn' as const, effectiveModel: 'model-b', + }, + invocationEvidence: { + executionId: 'execution-live', attemptId: 'attempt-old', modelChangeId: 'cross-attempt-model', + generation: 1, occurrenceId: 'model-occurrence', requestedModel: 'model-b', + effectiveModel: 'model-b', acceptedAt: base.createdAt, + }, + }; + assert.throws(() => decodeDurableGoalSessionState({ + ...base, status: 'running', currentModel: 'model-b', modelChangeGeneration: 1, + modelChangeIntents: [intent], modelChangeIntent: intent, + activeTurn: { + turnId: 'turn-live', executionId: 'execution-live', attemptId: 'attempt-new', executionEpoch: 1, + objective: 'objective', requestedModel: 'model-b', repository, status: 'running', + modelChange: { modelChangeId: intent.modelChangeId, generation: 1, previousModel: 'model-a' }, + }, + }), /activeTurn model invocation evidence/); +}); + +test('orphan pending lease-expiry poison fails before every provider mutation', async () => { + let providerMutations = 0; + const adapter: GoalSessionAdapter = { + provider: 'poison-adapter', + capabilities: { + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + }, + publishOperationBarrier: async () => { providerMutations += 1; }, + openSession: async () => { + providerMutations += 1; + return { providerSessionId: 'poison-native', recoveryMetadata: {} }; + }, + beginTurn: async function* () { providerMutations += 1; }, + resumeSession: async (_request, snapshot) => { providerMutations += 1; return snapshot; }, + requestModelChange: async request => { + providerMutations += 1; + return { requestedModel: request.model, appliesAt: 'next_turn' }; + }, + cancel: async () => { providerMutations += 1; }, + reconcile: async () => { providerMutations += 1; return { outcome: 'failed', reason: 'unused' }; }, + }; + const ports = new InMemoryGoalSessionPorts(); + const timestamp = new Date().toISOString(); + await ports.create({ + ...identity, provider: adapter.provider, providerSessionId: 'poison-native', recoveryMetadata: {}, + controllerEpoch: 1, status: 'idle', currentModel: 'model-a', completedTurnIds: [], + providerOperationGeneration: 4, + providerBarrierIntent: { + generation: 4, operationId: 'missing-live-intent:lease-expiry', kind: 'lease_expiry', + phase: 'pending', claimedAt: timestamp, + }, + createdAt: timestamp, updatedAt: timestamp, + }); + const before = await ports.load(identity); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await assert.rejects( + supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'INVALID_DURABLE_STATE', + ); + assert.equal(providerMutations, 0); + assert.deepEqual(await ports.load(identity), before); + assert.deepEqual(await ports.replay(identity), []); +}); + +test('all resolved provider DTO boundaries rebuild hostile proxies as one generic error', async () => { + const hostile = new Proxy({}, { + ownKeys() { throw new Error('docker run npm install ../../secret tcp://host unix:///socket C:\\credential'); }, + }); + const boundaries: Array<(value: unknown) => unknown> = [ + value => rebuildProviderSnapshot(value, 'codec-adapter'), + rebuildPauseAcknowledgement, + rebuildMessageAcknowledgement, + rebuildModelAcknowledgement, + value => rebuildReconcileResult(value, 'codec-adapter'), + rebuildIteratorResult, + ]; + for (const rebuild of boundaries) { + const error = await untrustedProviderResult(() => Promise.resolve(hostile), rebuild) + .catch(value => value as Error); + assert.equal(error.message, 'Provider operation failed safely'); + assert.equal((error as Error & { cause?: unknown }).cause, undefined); + assert.doesNotMatch(JSON.stringify(error), /docker|npm|secret|tcp|unix|credential/i); + } +}); + +test('provider recovery codecs are versioned, bounded, cross-provider closed, and usage-watermark stable', () => { + const envelopes = [ + { provider: 'codex', protocolVersion: 'app-server-0.146.0', payload: { threadId: 'thread-a', initialized: true } }, + { provider: 'claude', protocolVersion: 'cli-2.1.220', payload: { sessionId: 'session-a' } }, + { provider: 'antigravity', protocolVersion: 'cli-1.1.13', payload: { + conversationId: 'conversation-a', manifestVersion: 1, manifestChecksum: 'checksum-a', + } }, + ] as const; + for (const envelope of envelopes) { + assert.deepEqual(sanitizeRecoveryMetadata({ + version: 2, ...envelope, + usage: { components: [{ component: 'input_tokens', watermark: 4, occurrenceId: 'usage-4' }] }, + }, envelope.provider), { + version: 2, ...envelope, + usage: { components: [{ component: 'input_tokens', watermark: 4, occurrenceId: 'usage-4' }] }, + }); + } + assert.deepEqual(sanitizeRecoveryMetadata({ version: 1, checkpoint: 'legacy-safe', offset: 0 }), { + version: 1, checkpoint: 'legacy-safe', offset: 0, + }); + assert.throws(() => sanitizeRecoveryMetadata({ + version: 2, ...envelopes[0], usage: { components: [] }, + }, 'claude')); + assert.throws(() => sanitizeRecoveryMetadata({ + version: 2, provider: 'codex', protocolVersion: 'future', + payload: { threadId: 'thread-a', initialized: true }, usage: { components: [] }, + })); + assert.throws(() => sanitizeRecoveryMetadata({ + version: 2, ...envelopes[0], usage: { components: [ + { component: 'input_tokens', watermark: 1, occurrenceId: 'usage-1' }, + { component: 'input_tokens', watermark: 2, occurrenceId: 'usage-2' }, + ] }, + })); + assert.throws(() => sanitizeRecoveryMetadata({ checkpoint: 'x'.repeat(33 * 1024) })); + assert.throws(() => sanitizeNewRecoveryMetadata({}, 'codex')); + assert.throws(() => sanitizeNewRecoveryMetadata({ version: 1, checkpoint: 'legacy-safe' }, 'claude')); + assert.throws(() => sanitizeNewRecoveryMetadata({ + version: 2, ...envelopes[0], usage: { components: [] }, + }, 'codex'), /exact identity/); +}); + +class LineTransport { + readonly writes: Array> = []; + readonly output: AsyncIterable; + readonly completion = Promise.resolve({ exitCode: 0 }); + cancelled = false; + private ended = false; + private readonly lines: string[] = []; + private readonly readers: Array<(result: IteratorResult) => void> = []; + + constructor() { + this.output = { [Symbol.asyncIterator]: () => ({ next: () => this.next() }) }; + } + + async write(line: string): Promise { + const request = JSON.parse(line) as Record; + this.writes.push(request); + const id = request.id; + if (id === undefined) return; + const method = request.method; + if (method === 'initialize') this.push(JSON.stringify({ id, result: { + userAgent: 'propr_goal_runtime/0.146.0 (Linux; x86_64) test', codexHome: '/home/node/.codex', + platformFamily: 'unix', platformOs: 'linux', + } })); + else if (method === 'model/list') this.push(JSON.stringify({ + id, result: { data: [{ id: 'gpt-5.6-sol', model: 'gpt-5.6-sol' }], nextCursor: null }, + })); + else if (method === 'thread/start') this.push(JSON.stringify({ + id, result: threadResponse(false), + })); + else if (method === 'thread/resume') this.push(JSON.stringify({ + id, result: threadResponse(true), + })); + else throw new Error(`Unexpected test protocol method ${String(method)}`); + } + + closeInput(): void {} + async cancel(): Promise { this.cancelled = true; } + + private next(): Promise> { + const line = this.lines.shift(); + if (line !== undefined) return Promise.resolve({ done: false, value: line }); + if (this.ended) return Promise.resolve({ done: true, value: undefined }); + return new Promise(resolve => this.readers.push(resolve)); + } + + protected push(line: string): void { + const reader = this.readers.shift(); + if (reader) reader({ done: false, value: line }); + else this.lines.push(line); + } + + protected finish(): void { + this.ended = true; + for (const reader of this.readers.splice(0)) reader({ done: true, value: undefined }); + } +} + +class LostThreadStartTransport extends LineTransport { + override async write(line: string): Promise { + const request = JSON.parse(line) as Record; + if (request.method !== 'thread/start') return super.write(line); + this.writes.push(request); + this.finish(); + } +} + +class MalformedModelListTransport extends LineTransport { + override async write(line: string): Promise { + const request = JSON.parse(line) as Record; + if (request.method !== 'model/list') return super.write(line); + this.writes.push(request); + this.push(JSON.stringify({ id: request.id, result: {} })); + } +} + +function threadResponse(resume: boolean): Record { + return { + thread: { + id: 'codex-thread', extra: null, sessionId: 'codex-session', forkedFromId: null, + parentThreadId: null, preview: '', ephemeral: false, isPinned: false, + historyMode: 'paginated', modelProvider: 'openai', createdAt: 1, updatedAt: 1, + recencyAt: 1, status: { type: 'idle' }, path: null, cwd: '/workspace', + cliVersion: '0.146.0', source: 'appServer', canAcceptDirectInput: true, + threadSource: null, agentNickname: null, agentRole: null, gitInfo: null, name: null, turns: [], + }, + model: 'gpt-5.6-sol', modelProvider: 'openai', serviceTier: null, cwd: '/workspace', + runtimeWorkspaceRoots: ['/workspace'], instructionSources: [], approvalPolicy: 'never', + approvalsReviewer: 'user', sandbox: { + type: 'workspaceWrite', writableRoots: ['/workspace'], networkAccess: false, + excludeTmpdirEnvVar: false, excludeSlashTmp: false, + }, + activePermissionProfile: null, reasoningEffort: null, multiAgentMode: 'explicitRequestOnly', + ...(resume ? { initialTurnsPage: null, turnsBackwardsCursor: null, itemsBackwardsCursor: null } : {}), + }; +} + +test('supervised Codex eager open uses stdio, exact gpt-5.6-sol, and starts no fake turn', async () => { + const transport = new LineTransport(); + const context: GoalProviderOpenContext = { + executionId: 'codex-execution', attemptId: 'codex-attempt', repository, + requestedModel: 'gpt-5.6-sol', providerHomeTarget: '/home/node/.codex', + credentialTargets: ['/home/node/.codex/auth.json'], deterministicOpenKey: 'durable-open-key', transport, + }; + const snapshot = await openSupervisedCodexAppServer(context); + assert.equal(snapshot.providerSessionId, 'codex-thread'); + assert.equal(snapshot.model, 'gpt-5.6-sol'); + assert.deepEqual(transport.writes.map(write => write.method), [ + 'initialize', 'initialized', 'model/list', 'thread/start', + ]); + assert.equal('params' in transport.writes[1], false); + assert.deepEqual((transport.writes[0].params as Record).capabilities, { + experimentalApi: false, requestAttestation: false, + }); + assert.equal(transport.writes.some(write => write.method === 'turn/start'), false); + const start = transport.writes.find(write => write.method === 'thread/start'); + assert.equal((start?.params as Record)?.model, 'gpt-5.6-sol'); + assert.equal((start?.params as Record)?.cwd, '/workspace'); + assert.equal((start?.params as Record)?.approvalPolicy, 'never'); + assert.equal((start?.params as Record)?.sandbox, 'workspace-write'); + assert.equal('serviceName' in (start?.params as Record), false); + assert.equal('metadata' in (start?.params as Record), false); + assert.deepEqual(sanitizeRecoveryMetadata(snapshot.recoveryMetadata, 'codex'), snapshot.recoveryMetadata); + assert.equal(transport.cancelled, true, 'successful open explicitly closes the owned App Server transport'); +}); + +test('Codex response loss fails closed and persisted exact identity is the only resume path', async () => { + const first = new LineTransport(); + const context: GoalProviderOpenContext = { + executionId: 'codex-execution', attemptId: 'codex-attempt', repository, + requestedModel: 'gpt-5.6-sol', providerHomeTarget: '/home/node/.codex', + credentialTargets: [], deterministicOpenKey: 'durable-open-key', transport: first, + }; + const persisted = await openSupervisedCodexAppServer(context); + const lost = new LostThreadStartTransport(); + await assert.rejects( + openSupervisedCodexAppServer({ ...context, transport: lost }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'PROVIDER_OPEN_IN_DOUBT', + ); + assert.equal(lost.writes.some(write => write.method === 'thread/list'), false); + assert.equal(lost.cancelled, true); + + const resumed = new LineTransport(); + const snapshot = await openSupervisedCodexAppServer({ ...context, transport: resumed }, persisted); + assert.equal(snapshot.providerSessionId, 'codex-thread'); + assert.equal(resumed.writes.some(write => write.method === 'thread/start'), false); + assert.equal(resumed.writes.some(write => write.method === 'thread/resume'), true); + + const mutations: Array<(snapshot: GoalProviderSessionSnapshot) => void> = [ + snapshot => { snapshot.providerSessionId = 'foreign-thread'; }, + snapshot => { snapshot.model = 'different-model'; }, + snapshot => { (snapshot.recoveryMetadata as { protocolVersion: string }).protocolVersion = 'future'; }, + snapshot => { ((snapshot.recoveryMetadata as { payload: Record }).payload).openKey = 'other-key'; }, + snapshot => { ((snapshot.recoveryMetadata as { payload: Record }).payload).repository = 'other/repo'; }, + snapshot => { ((snapshot.recoveryMetadata as { payload: Record }).payload).model = 'different-model'; }, + snapshot => { ((snapshot.recoveryMetadata as { payload: Record }).payload).providerHomeIdentity = '/other'; }, + snapshot => { ((snapshot.recoveryMetadata as { payload: Record }).payload).cliVersion = '0.145.0'; }, + ]; + for (const mutate of mutations) { + const mismatched = structuredClone(persisted); + mutate(mismatched); + const rejected = new LineTransport(); + await assert.rejects( + openSupervisedCodexAppServer({ ...context, transport: rejected }, mismatched), + (error: unknown) => error instanceof GoalSessionContractError, + ); + assert.equal(rejected.writes.some(write => write.method === 'thread/resume'), false); + } + + const malformed = new MalformedModelListTransport(); + await assert.rejects(openSupervisedCodexAppServer({ ...context, transport: malformed }), + /Codex App Server open failed safely/); + assert.equal(malformed.writes.some(write => write.method === 'thread/start'), false); +}); + +test('production Supervisor crash/reopen issues one total Codex thread/start and keeps durable doubt terminal', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-response-loss-supervisor-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const seedDatabase = new Database(filename); + seedAuthoritativeGoal(seedDatabase, { goalId: identity.goalId, agent: 'codex', model: 'gpt-5.6-sol' }); + seedDatabase.close(); + const transports: LineTransport[] = []; + const plan = issueGoalSupervisedOpenPlan({ + repository, requestedModel: 'gpt-5.6-sol', providerHomeTarget: '/home/node/.codex', credentialTargets: [], + }, { + createTransport: async () => { + const transport = new LostThreadStartTransport(); + transports.push(transport); + return transport; + }, + cancelPending: async () => undefined, + transferPending: () => undefined, + }); + const adapter: GoalSessionAdapter = { + provider: 'codex', supportsDeterministicOpen: true, + capabilities: { nativeSessionId: 'eager', steering: 'active_turn', pause: 'after_turn', modelChange: 'next_turn' }, + publishOperationBarrier: async () => undefined, + openSession: request => openSupervisedCodexAppServer(request.openContext!, request.persisted), + beginTurn: async function* () { yield { type: 'completion', outcome: 'succeeded' }; }, + resumeSession: async (_request, snapshot) => snapshot, + requestModelChange: async request => ({ requestedModel: request.model, appliesAt: 'next_turn' }), + cancel: async () => undefined, cancelPending: async () => undefined, + reconcile: async () => ({ outcome: 'failed', reason: 'unused' }), + }; + let database = new Database(filename); + const first = new GoalSessionSupervisor( + adapter, createSqliteGoalSessionRuntimePorts(database, recovery), () => 'codex-response-attempt', + ); + await assert.rejects(first.openSession({ ...identity, provider: 'codex', controllerEpoch: 1, supervisedOpen: plan }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'PROVIDER_OPEN_IN_DOUBT'); + database.close(); + database = new Database(filename); + const runtime = createSqliteGoalSessionRuntimePorts(database, recovery); + const replacement = new GoalSessionSupervisor(adapter, runtime, () => 'codex-replacement-attempt'); + await assert.rejects(replacement.openSession({ + ...identity, provider: 'codex', controllerEpoch: 2, supervisedOpen: plan, + }), /failed provider session cannot be resumed/); + assert.equal(transports.length, 1); + assert.equal(transports.flatMap(transport => transport.writes).filter(write => write.method === 'thread/start').length, 1); + assert.equal((await runtime.state.load(identity))?.status, 'failed'); + database.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +test('hardened supervisor constructs eager-open transport only under its exact durable control claim', async () => { + const ports = new InMemoryGoalSessionPorts(); + const adapter: GoalSessionAdapter = { + provider: 'codex', + capabilities: { + nativeSessionId: 'eager', steering: 'active_turn', pause: 'after_turn', modelChange: 'next_turn', + }, + supportsDeterministicOpen: true, + publishOperationBarrier: async () => undefined, + openSession: async request => { + assert.ok(request.openContext); + assert.equal('turnId' in request.openContext, false); + return openSupervisedCodexAppServer(request.openContext); + }, + beginTurn: async function* () { yield { type: 'completion', outcome: 'succeeded' }; }, + resumeSession: async (_request, snapshot) => snapshot, + requestModelChange: async request => ({ + requestedModel: request.model, appliesAt: 'next_turn', effectiveModel: request.model, + }), + cancel: async () => undefined, + reconcile: async () => ({ outcome: 'failed', reason: 'unused provider prose' }), + }; + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + const transport = new LineTransport(); + let factoryCalled = false; + const opened = await supervisor.openSession({ + ...identity, provider: 'codex', controllerEpoch: 1, + supervisedOpen: issueGoalSupervisedOpenPlan({ + repository, requestedModel: 'gpt-5.6-sol', providerHomeTarget: '/home/node/.codex', + credentialTargets: ['/home/node/.codex/auth.json'], + }, { + createTransport: async claim => { + factoryCalled = true; + const durable = await ports.load(identity); + assert.equal(durable?.providerOpenAttemptId, claim.attemptId); + assert.equal(durable?.providerOperationGeneration, claim.operationGeneration); + assert.equal(claim.operationFence.goalId, identity.goalId); + assert.equal(claim.operationFence.sessionId, identity.sessionId); + assert.equal(claim.operationFence.generation, claim.operationGeneration); + assert.equal(claim.operationFence.operationId, claim.attemptId); + assert.equal(claim.operationFence.kind, 'open'); + assert.equal(claim.operationFence.leaseExpiresAt, undefined); + assert.equal('turnId' in claim, false); + assert.match(claim.deterministicOpenKey, /^[A-Za-z0-9._:-]+$/); + return transport; + }, + cancelPending: async () => { await transport.cancel(); }, + transferPending: () => undefined, + }), + }); + assert.equal(factoryCalled, true); + assert.equal(opened.status, 'idle'); + assert.equal(opened.providerSessionId, 'codex-thread'); + assert.equal(opened.currentModel, 'gpt-5.6-sol'); +}); + +class NextTurnEvidenceAdapter implements GoalSessionAdapter { + readonly provider = 'next-turn-evidence'; + readonly capabilities = { + nativeSessionId: 'eager' as const, steering: 'next_turn' as const, + pause: 'after_turn' as const, modelChange: 'next_turn' as const, + }; + mode: 'duplicate' | 'missing' | 'wrong' = 'duplicate'; + betweenDuplicates?: () => Promise; + async publishOperationBarrier(): Promise {} + async openSession(): Promise { + return { providerSessionId: 'evidence-native', recoveryMetadata: {}, model: 'model-a' }; + } + async *beginTurn(request: GoalBeginTurnRequest): AsyncIterable { + if (this.mode !== 'missing' && request.modelChange) { + const event = { + type: 'model_changed' as const, + model: this.mode === 'wrong' ? 'model-wrong' : request.requestedModel, + providerEventId: `model-${request.modelChange.modelChangeId}-${request.modelChange.generation}`, + }; + yield event; + if (this.mode === 'duplicate') { + await this.betweenDuplicates?.(); + yield event; + } + } + yield { type: 'completion', outcome: 'succeeded' }; + } + async resumeSession(_request: never, snapshot: GoalProviderSessionSnapshot) { return snapshot; } + async requestModelChange(request: GoalModelChangeRequest) { + return { requestedModel: request.model, appliesAt: 'next_turn' as const }; + } + async cancel(): Promise {} + async reconcile() { return { outcome: 'failed' as const, reason: 'unused' }; } +} + +test('next-turn model evidence dedupes one occurrence and withholds unproven completion', async t => { + for (const mode of ['duplicate', 'missing', 'wrong'] as const) { + await t.test(mode, async () => { + const adapter = new NextTurnEvidenceAdapter(); + adapter.mode = mode; + const ports = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + await supervisor.requestModelChange({ ...identity, controllerEpoch: 1, model: 'model-b' }); + if (mode === 'duplicate') adapter.betweenDuplicates = async () => { + const state = await ports.load(identity); + const evidence = state?.modelChangeIntent?.invocationEvidence; + assert.deepEqual(evidence && { + executionId: evidence.executionId, attemptId: evidence.attemptId, + occurrenceId: evidence.occurrenceId, effectiveModel: evidence.effectiveModel, + }, { + executionId: 'execution-duplicate', attemptId: 'attempt-duplicate', + occurrenceId: `model-${state?.modelChangeIntent?.modelChangeId}-1`, effectiveModel: 'model-b', + }); + }; + const operation = supervisor.runTurn({ + ...identity, controllerEpoch: 1, turnId: `evidence-${mode}`, + executionId: `execution-${mode}`, attemptId: `attempt-${mode}`, + objective: 'require exact next-turn evidence', repository, requestedModel: 'model-a', + }); + if (mode === 'duplicate') { + assert.equal((await operation).state.status, 'idle'); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); + return; + } + await assert.rejects(operation, (error: unknown) => error instanceof GoalSessionContractError + && (error.code === 'MODEL_EVIDENCE_MISSING' || error.code === 'MODEL_ACK_MISMATCH')); + assert.equal((await ports.load(identity))?.status, 'failed'); + assert.equal((await ports.replay(identity)).some(record => + record.event.type === 'completion' && record.event.outcome === 'succeeded'), false); + }); + } +}); + +class UsageAdapter implements GoalSessionAdapter { + readonly provider = 'usage-adapter'; + readonly capabilities = { + nativeSessionId: 'eager' as const, steering: 'next_turn' as const, + pause: 'after_turn' as const, modelChange: 'next_turn' as const, + }; + events: GoalSessionEvent[] = []; + async publishOperationBarrier(): Promise {} + async openSession(_request: GoalProviderOpenRequest): Promise { + return { providerSessionId: 'usage-native', recoveryMetadata: {}, model: 'model-a' }; + } + async *beginTurn(_request: GoalBeginTurnRequest): AsyncIterable { yield* this.events; } + async resumeSession(_request: never, snapshot: GoalProviderSessionSnapshot) { return snapshot; } + async requestModelChange() { return { requestedModel: 'model-a', appliesAt: 'next_turn' as const }; } + async cancel(_request: GoalProviderCancelRequest): Promise {} + async reconcile() { return { outcome: 'failed' as const, reason: 'unused' }; } +} + +function turn(turnId: string) { + return { + ...identity, controllerEpoch: 1, turnId, executionId: `execution-${turnId}`, attemptId: `attempt-${turnId}`, + objective: 'account usage exactly once', repository, requestedModel: 'model-a', + }; +} + +test('usage occurrences dedupe across replay while cumulative watermarks advance monotonically', async () => { + const ports = new InMemoryGoalSessionPorts(); + const adapter = new UsageAdapter(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + adapter.events = [ + { type: 'usage', occurrenceId: 'usage-0', semantics: 'delta', watermark: 0, inputTokens: 3 }, + { type: 'usage', occurrenceId: 'usage-0', semantics: 'delta', watermark: 0, inputTokens: 3 }, + { type: 'usage', occurrenceId: 'usage-2', semantics: 'cumulative', watermark: 2, inputTokens: 5 }, + { type: 'completion', outcome: 'succeeded' }, + ]; + await supervisor.runTurn(turn('one')); + adapter.events = [ + { type: 'usage', occurrenceId: 'usage-2', semantics: 'cumulative', watermark: 2, inputTokens: 5 }, + { type: 'usage', occurrenceId: 'usage-3', semantics: 'cumulative', watermark: 3, inputTokens: 8 }, + { type: 'completion', outcome: 'succeeded' }, + ]; + await supervisor.runTurn(turn('two')); + const state = await ports.load(identity); + assert.deepEqual(state?.usageAccounting, { + version: 1, lastWatermark: 3, occurrences: ['usage-0', 'usage-2', 'usage-3'], + }); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'usage').length, 3); +}); + +test('pending cancellation barrier is replayed with its exact identity before any reopen work', async () => { + class ReplayAdapter extends UsageAdapter { + failPublication = false; + readonly publications: Array<{ generation: number; pendingCancellationId?: string }> = []; + readonly cancellations: string[] = []; + override async publishOperationBarrier(publication: { generation: number; pendingCancellationId?: string }) { + this.publications.push(structuredClone(publication)); + if (this.failPublication) throw new Error('untrusted barrier details'); + } + override async cancel(request: GoalProviderCancelRequest): Promise { + this.cancellations.push(request.cancellationId); + } + } + const ports = new InMemoryGoalSessionPorts(); + const firstAdapter = new ReplayAdapter(); + const first = new GoalSessionSupervisor(firstAdapter, ports.asRuntimePorts()); + await first.openSession({ ...identity, provider: firstAdapter.provider, controllerEpoch: 1 }); + firstAdapter.failPublication = true; + await assert.rejects(first.cancel({ ...identity, controllerEpoch: 1, reason: 'durable replay' }), + /Provider barrier publication failed safely/); + const pending = await ports.load(identity); + assert.equal(pending?.providerBarrierIntent?.phase, 'pending'); + const cancellationId = pending?.cancellationIntent?.cancellationId; + assert.ok(cancellationId); + + const replacementAdapter = new ReplayAdapter(); + const reopened = await new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()).openSession({ + ...identity, provider: replacementAdapter.provider, controllerEpoch: 2, + }); + assert.equal(reopened.status, 'terminated'); + assert.deepEqual(replacementAdapter.cancellations, [cancellationId]); + assert.ok(replacementAdapter.publications.some(publication => + publication.pendingCancellationId === cancellationId)); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('same-controller cancel retry repairs an exact pending terminal barrier', async () => { + class TerminalRepairAdapter extends UsageAdapter { + terminalPublicationFails = true; + override async publishOperationBarrier(publication: { generation: number; pendingCancellationId?: string }) { + if (this.terminalPublicationFails && publication.generation >= 3) { + throw new Error('hostile terminal publication detail'); + } + } + } + const ports = new InMemoryGoalSessionPorts(); + const adapter = new TerminalRepairAdapter(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + await assert.rejects(supervisor.cancel({ ...identity, controllerEpoch: 1, reason: 'repair terminal' }), + /Provider barrier publication failed safely/); + assert.equal((await ports.load(identity))?.status, 'terminated'); + assert.equal((await ports.load(identity))?.providerBarrierIntent?.phase, 'pending'); + adapter.terminalPublicationFails = false; + const repaired = await supervisor.cancel({ ...identity, controllerEpoch: 1, reason: 'retry' }); + assert.equal(repaired.status, 'terminated'); + assert.equal(repaired.providerBarrierIntent?.phase, 'published'); + assert.equal((await ports.replay(identity)).filter(event => event.event.type === 'completion').length, 1); +}); diff --git a/packages/core/test/goalSessionExactHeadReaudit.test.ts b/packages/core/test/goalSessionExactHeadReaudit.test.ts new file mode 100644 index 000000000..8401c0de0 --- /dev/null +++ b/packages/core/test/goalSessionExactHeadReaudit.test.ts @@ -0,0 +1,808 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalProviderModelChangeRequest, + GoalProviderOpenRequest, + GoalProviderReconcileRequest, + GoalSessionAdapter, + GoalSessionControlTransition, + GoalSessionEvent, + GoalSessionRuntimePorts, + GoalSessionState, + GoalTerminalCommit, +} from '../src/agents/goalSession/contract.js'; +import { + GoalSessionSupervisor, + StaleGoalSessionFenceError, +} from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { streamAuditTransitionId } from '../src/agents/goalSession/turnStreamProtocol.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; +import { SqliteGoalSessionTestPorts } from './SqliteGoalSessionTestPorts.js'; + +const identity = { goalId: 'exact-head-goal', sessionId: 'exact-head-session' }; +const control = { ...identity, controllerEpoch: 1 }; +const fence = { ...control, turnId: 'exact-head-turn' }; +const repository = { + repository: 'integry/propr', worktreePath: '/tmp/exact-head', branch: 'reaudit', headSha: '20e53540', +}; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +function sqlitePersistence(): { filename: string; cleanup: () => void } { + const directory = mkdtempSync(join(tmpdir(), 'goal-session-cross-process-')); + return { filename: join(directory, 'goal-session.sqlite'), cleanup: () => rmSync(directory, { recursive: true, force: true }) }; +} + +type Effects = { model: string; calls: GoalProviderModelChangeRequest[] }; + +class ExactHeadAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'exact-head-provider'; + readonly capabilities = { + nativeSessionId: 'eager' as const, + steering: 'active_turn' as const, + pause: 'active_turn' as const, + modelChange: 'next_safe_boundary' as const, + }; + stream: () => AsyncIterable = async function* () { + yield { type: 'completion', outcome: 'succeeded' }; + }; + modelGates = new Map>(); + modelStarted = new Map void>(); + modelFailures = new Set(); + reconcileGate: Promise | undefined; + reconcileStarted: (() => void) | undefined; + reconcileCalls = 0; + reconcileFailure = false; + cancelCalls = 0; + + constructor(readonly effects: Effects = { model: 'model-a', calls: [] }) {} + + async openSession(_request: GoalProviderOpenRequest) { + return { providerSessionId: 'exact-native', recoveryMetadata: { checkpoint: 'open' }, model: this.effects.model }; + } + + beginTurn(_request: GoalBeginTurnRequest) { return this.stream(); } + + async resumeSession(_request: typeof control, snapshot: { providerSessionId: string; recoveryMetadata: unknown }) { + return snapshot; + } + + async requestModelChange(request: GoalProviderModelChangeRequest) { + this.effects.calls.push(structuredClone(request)); + this.modelStarted.get(request.model)?.(); + const gate = this.modelGates.get(request.model); + if (gate) await gate; + this.effects.model = request.model; + if (this.modelFailures.has(request.model)) throw new Error(`local failure after applying ${request.model}`); + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + + async cancel() { this.cancelCalls += 1; } + + async reconcile(_request: GoalProviderReconcileRequest) { + this.reconcileCalls += 1; + this.reconcileStarted?.(); + if (this.reconcileGate) await this.reconcileGate; + if (this.reconcileFailure) throw new Error('reconcile transport failed'); + return { + outcome: 'resumed' as const, + snapshot: { providerSessionId: 'exact-native', recoveryMetadata: { checkpoint: 'reconciled' }, model: this.effects.model }, + reason: 'resumed exact invocation', + }; + } +} + +async function opened( + adapter = new ExactHeadAdapter(), + ports: { asRuntimePorts(): GoalSessionRuntimePorts } = new InMemoryGoalSessionPorts(), +) { + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + return { adapter, ports, supervisor }; +} + +function turnRequest() { + return { + ...fence, + executionId: 'exact-execution', + attemptId: 'exact-attempt', + objective: 'exercise exact-head invariants', + repository, + requestedModel: 'model-a', + }; +} + +test('stream transition identity is exact-attempt scoped and occurrence stable', async () => { + const adapter = new ExactHeadAdapter(); + adapter.stream = async function* () { + yield { type: 'model_changed', previousModel: 'model-a', model: 'model-b', providerEventOrdinal: 1 }; + yield { type: 'model_changed', previousModel: 'model-a', model: 'model-b', providerEventOrdinal: 1 }; + yield { type: 'model_changed', previousModel: 'model-b', model: 'model-c', providerEventOrdinal: 2 }; + yield { type: 'model_changed', previousModel: 'model-c', model: 'model-b', providerEventOrdinal: 3 }; + yield { type: 'completion', outcome: 'succeeded' }; + }; + const { ports, supervisor } = await opened(adapter); + await supervisor.runTurn(turnRequest()); + const models = (await ports.replay(identity)).flatMap(record => + record.event.type === 'model_changed' ? [record.event.model] : []); + assert.deepEqual(models, ['model-b', 'model-c', 'model-b']); + + const stale = await ports.load(identity); + assert.ok(stale); + const oldExecution = { executionId: 'old-execution', attemptId: 'old-attempt' }; + const event = { type: 'model_changed' as const, model: 'model-z', providerEventId: 'provider-event-z' }; + const oldId = streamAuditTransitionId(fence, oldExecution, event); + assert.notEqual(oldId, streamAuditTransitionId(fence, { ...oldExecution, attemptId: 'new-attempt' }, event)); +}); + +test('provider event IDs have deterministic precedence and never mix mutable payload or local ordinal', () => { + const execution = { executionId: 'stable-execution', attemptId: 'stable-attempt' }; + const first = { + type: 'model_changed' as const, + previousModel: 'model-a', + model: 'model-b', + providerEventId: 'provider-occurrence', + providerEventOrdinal: 1, + }; + const replay = { + ...first, + previousModel: 'mutated-predecessor', + model: 'mutated-payload', + providerEventOrdinal: 999, + }; + assert.equal(streamAuditTransitionId(fence, execution, first), + streamAuditTransitionId(fence, execution, replay)); + assert.throws(() => streamAuditTransitionId(fence, execution, { + type: 'pause_boundary', boundary: 'missing-identity', + }), /stable providerEventId or providerEventOrdinal/); + assert.throws(() => streamAuditTransitionId(fence, execution, { + type: 'model_changed', model: 'model-b', providerEventId: ' ', providerEventOrdinal: 1, + }), /providerEventId must be non-empty/); +}); + +test('missing streamed occurrence identity fails closed before model state or audit mutation', async () => { + const adapter = new ExactHeadAdapter(); + adapter.stream = async function* () { yield { type: 'model_changed', model: 'model-b' }; }; + const { ports, supervisor } = await opened(adapter); + await assert.rejects(supervisor.runTurn(turnRequest()), /stable providerEventId or providerEventOrdinal/); + const state = await ports.load(identity); + assert.equal(state?.currentModel, 'model-a'); + assert.equal(state?.status, 'failed'); + const events = await ports.replay(identity); + assert.equal(events.filter(record => record.event.type === 'model_changed').length, 0); + assert.equal(events.filter(record => record.event.type === 'completion').length, 1); +}); + +test('transition dedupe validates the live exact attempt before returning an old hit', async () => { + const ports = new InMemoryGoalSessionPorts(); + const timestamp = new Date().toISOString(); + const oldExecution = { executionId: 'execution-one', attemptId: 'attempt-one' }; + const state = await ports.create({ + ...control, + provider: 'exact-head-provider', + providerSessionId: 'exact-native', + recoveryMetadata: {}, + status: 'running', + currentModel: 'model-a', + completedTurnIds: [], + activeTurn: { + ...oldExecution, turnId: fence.turnId, executionEpoch: 1, objective: 'transition', + requestedModel: 'model-a', repository, status: 'running', + }, + createdAt: timestamp, + updatedAt: timestamp, + }); + assert.ok(state); + const event = { type: 'model_changed' as const, model: 'model-b', providerEventOrdinal: 7 }; + const transition: GoalSessionControlTransition = { + transitionId: streamAuditTransitionId(fence, oldExecution, event), + fence, + turnScoped: true, + execution: oldExecution, + auditEvents: [event], + }; + const next = { ...state, currentModel: 'model-b' }; + delete (next as Partial).version; + const committed = await ports.commit(state, next, transition); + assert.ok(committed); + assert.ok(await ports.commit(state, next, transition), 'exact duplicate redelivery dedupes'); + + const newExecution = { executionId: 'execution-one', attemptId: 'attempt-two' }; + const replaced = await ports.compareAndSet(committed, { + ...committed, + activeTurn: committed.activeTurn ? { ...committed.activeTurn, ...newExecution } : undefined, + }); + assert.ok(replaced); + assert.equal(await ports.commit(state, next, transition), null, 'stale old-attempt hit must fail its live fence'); + + const recoveredTransition = { + ...transition, + execution: newExecution, + transitionId: streamAuditTransitionId(fence, newExecution, event), + }; + const recoveredNext = { ...replaced, currentModel: 'model-b' }; + delete (recoveredNext as Partial).version; + assert.ok(await ports.commit(replaced, recoveredNext, recoveredTransition), + 'a recovered fresh attempt may commit the same semantic event'); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'model_changed').length, 2); +}); + +test('exact streamed occurrence redelivery dedupes after SQLite-backed process reopen', async t => { + const persistence = sqlitePersistence(); + const first = new SqliteGoalSessionTestPorts(persistence.filename); + const timestamp = new Date().toISOString(); + const execution = { executionId: 'sqlite-execution', attemptId: 'sqlite-attempt' }; + const state = await first.create({ + ...control, + provider: 'exact-head-provider', + providerSessionId: 'exact-native', + recoveryMetadata: {}, + status: 'running', + currentModel: 'model-a', + completedTurnIds: [], + activeTurn: { + ...execution, turnId: fence.turnId, executionEpoch: 1, objective: 'sqlite replay', + requestedModel: 'model-a', repository, status: 'running', + }, + createdAt: timestamp, + updatedAt: timestamp, + }); + assert.ok(state); + const event = { type: 'model_changed' as const, model: 'model-b', providerEventId: 'sqlite-occurrence' }; + const transition: GoalSessionControlTransition = { + transitionId: streamAuditTransitionId(fence, execution, event), + fence, turnScoped: true, execution, auditEvents: [event], + }; + const next = { ...state, currentModel: 'model-b' }; + delete (next as Partial).version; + assert.ok(await first.commit(state, next, transition)); + first.close(); + const reopened = new SqliteGoalSessionTestPorts(persistence.filename); + t.after(() => { reopened.close(); persistence.cleanup(); }); + assert.ok(await reopened.commit(state, next, transition)); + assert.equal((await reopened.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); +}); + +test('overlapping model generations converge after reverse completion and cached newest waits for repair', async () => { + const adapter = new ExactHeadAdapter(); + const oldGate = deferred(); + const oldStarted = deferred(); + adapter.modelGates.set('model-b', oldGate.promise); + adapter.modelStarted.set('model-b', oldStarted.resolve); + const { ports, supervisor } = await opened(adapter); + const old = supervisor.requestModelChange({ ...control, model: 'model-b' }); + await oldStarted.promise; + await supervisor.requestModelChange({ ...control, model: 'model-c' }); + + let repeatedSettled = false; + const repeated = supervisor.requestModelChange({ ...control, model: 'model-c' }) + .then(value => { repeatedSettled = true; return value; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(repeatedSettled, false); + oldGate.resolve(); + await assert.rejects(old, StaleGoalSessionFenceError); + assert.equal((await repeated).effectiveModel, 'model-c'); + assert.equal(adapter.effects.model, 'model-c'); + assert.equal((await ports.load(identity))?.currentModel, 'model-c'); + const changed = (await ports.replay(identity)).filter(record => record.event.type === 'model_changed'); + assert.deepEqual(changed.map(record => record.event.type === 'model_changed' ? record.event.model : ''), ['model-c']); + const acknowledgements = (await ports.replay(identity)) + .filter(record => record.event.type === 'model_change_acknowledged'); + assert.equal(acknowledgements.length, 2, 'each retained model intent has exactly one canonical acknowledgement'); + assert.equal(new Set(adapter.effects.calls.filter(call => call.model === 'model-c').map(call => call.modelChangeId)).size, 1); +}); + +test('three mixed-order model generations leave the provider and durable state on the newest intent', async () => { + const adapter = new ExactHeadAdapter(); + const gates = [deferred(), deferred()]; + const starts = [deferred(), deferred()]; + adapter.modelGates.set('model-b', gates[0].promise); + adapter.modelGates.set('model-c', gates[1].promise); + adapter.modelStarted.set('model-b', starts[0].resolve); + adapter.modelStarted.set('model-c', starts[1].resolve); + const { ports, supervisor } = await opened(adapter); + const first = supervisor.requestModelChange({ ...control, model: 'model-b' }); + await starts[0].promise; + const second = supervisor.requestModelChange({ ...control, model: 'model-c' }); + await starts[1].promise; + await supervisor.requestModelChange({ ...control, model: 'model-d' }); + gates[1].resolve(); + await assert.rejects(second, StaleGoalSessionFenceError); + gates[0].resolve(); + await assert.rejects(first, StaleGoalSessionFenceError); + assert.equal(adapter.effects.model, 'model-d'); + assert.equal((await ports.load(identity))?.currentModel, 'model-d'); + const changes = (await ports.replay(identity)).filter(record => record.event.type === 'model_changed'); + assert.deepEqual(changes.map(record => record.event.type === 'model_changed' ? record.event.model : ''), ['model-d']); +}); + +test('a stale model completion after process replacement repairs the newest durable generation', async () => { + const effects: Effects = { model: 'model-a', calls: [] }; + const adapter = new ExactHeadAdapter(effects); + const gate = deferred(); + const started = deferred(); + adapter.modelGates.set('model-b', gate.promise); + adapter.modelStarted.set('model-b', started.resolve); + const { ports, supervisor } = await opened(adapter); + const stale = supervisor.requestModelChange({ ...control, model: 'model-b' }); + await started.promise; + await supervisor.requestModelChange({ ...control, model: 'model-c' }); + + const replacementAdapter = new ExactHeadAdapter(effects); + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + const reopening = replacement.openSession({ + ...identity, provider: replacementAdapter.provider, controllerEpoch: 2, + }); + gate.resolve(); + await assert.rejects(stale, StaleGoalSessionFenceError); + assert.equal((await reopening).currentModel, 'model-c'); + assert.equal(effects.model, 'model-c'); + const acknowledgement = await replacement.requestModelChange({ ...identity, controllerEpoch: 2, model: 'model-c' }); + assert.equal(acknowledgement.effectiveModel, 'model-c'); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); +}); + +test('cached newest model waits across separate SQLite ports sharing only durable persistence', async t => { + const persistence = sqlitePersistence(); + const firstPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const secondPorts = new SqliteGoalSessionTestPorts(persistence.filename); + t.after(() => { firstPorts.close(); secondPorts.close(); persistence.cleanup(); }); + const effects: Effects = { model: 'model-a', calls: [] }; + const firstAdapter = new ExactHeadAdapter(effects); + const oldGate = deferred(); + const oldStarted = deferred(); + firstAdapter.modelGates.set('model-b', oldGate.promise); + firstAdapter.modelStarted.set('model-b', oldStarted.resolve); + const { supervisor } = await opened(firstAdapter, firstPorts); + const stale = supervisor.requestModelChange({ ...control, model: 'model-b' }); + await oldStarted.promise; + await supervisor.requestModelChange({ ...control, model: 'model-c' }); + + const replacement = new GoalSessionSupervisor(new ExactHeadAdapter(effects), secondPorts.asRuntimePorts()); + let cachedSettled = false; + const cached = replacement.requestModelChange({ ...control, model: 'model-c' }) + .then(value => { cachedSettled = true; return value; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(cachedSettled, false); + oldGate.resolve(); + await assert.rejects(stale, StaleGoalSessionFenceError); + assert.equal((await cached).effectiveModel, 'model-c'); + assert.equal(effects.model, 'model-c'); + assert.equal((await secondPorts.replay(identity)).filter(event => event.event.type === 'model_changed').length, 1); +}); + +test('reopen repairs a superseded provider-success/local-failure window before using cached newest state', async () => { + const effects: Effects = { model: 'model-a', calls: [] }; + const adapter = new ExactHeadAdapter(effects); + const gate = deferred(); + const started = deferred(); + adapter.modelGates.set('model-b', gate.promise); + adapter.modelStarted.set('model-b', started.resolve); + adapter.modelFailures.add('model-b'); + const { ports, supervisor } = await opened(adapter); + const stale = supervisor.requestModelChange({ ...control, model: 'model-b' }); + await started.promise; + await supervisor.requestModelChange({ ...control, model: 'model-c' }); + gate.resolve(); + await assert.rejects(stale, /Provider operation failed safely/); + assert.equal(effects.model, 'model-b'); + + const replacementAdapter = new ExactHeadAdapter(effects); + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + const reopened = await replacement.openSession({ + ...identity, provider: replacementAdapter.provider, controllerEpoch: 2, + }); + assert.equal(reopened.currentModel, 'model-c'); + assert.equal(effects.model, 'model-c'); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); + assert.equal(reopened.modelChangeIntents?.[0].phase, 'superseded'); +}); + +test('expired stale model lease leaves recovery evidence and cached newest repairs it cross-process', async t => { + const persistence = sqlitePersistence(); + const firstPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const effects: Effects = { model: 'model-a', calls: [] }; + const adapter = new ExactHeadAdapter(effects); + const gate = deferred(); + const started = deferred(); + adapter.modelGates.set('model-b', gate.promise); + adapter.modelStarted.set('model-b', started.resolve); + adapter.modelFailures.add('model-b'); + const { supervisor } = await opened(adapter, firstPorts); + const stale = supervisor.requestModelChange({ ...control, model: 'model-b' }); + await started.promise; + await supervisor.requestModelChange({ ...control, model: 'model-c' }); + gate.resolve(); + await assert.rejects(stale, /Provider operation failed safely/); + assert.equal(effects.model, 'model-b'); + + const evidence = await firstPorts.load(identity); + assert.ok(evidence?.modelChangeIntents?.[0].applicationToken); + const staleIntent = evidence.modelChangeIntents[0]; + const expiredIntents = evidence.modelChangeIntents.map(intent => + intent.modelChangeId === staleIntent.modelChangeId + ? { ...intent, leaseExpiresAt: new Date(0).toISOString() } + : intent); + assert.ok(await firstPorts.compareAndSet(evidence, { + ...evidence, + modelChangeIntents: expiredIntents, + modelChangeIntent: expiredIntents.at(-1), + })); + const replacementPorts = new SqliteGoalSessionTestPorts(persistence.filename); + t.after(() => { firstPorts.close(); replacementPorts.close(); persistence.cleanup(); }); + const replacement = new GoalSessionSupervisor( + new ExactHeadAdapter(effects), replacementPorts.asRuntimePorts(), + ); + assert.equal((await replacement.requestModelChange({ ...control, model: 'model-c' })).effectiveModel, 'model-c'); + assert.equal(effects.model, 'model-c'); + assert.equal((await replacementPorts.load(identity))?.modelChangeIntents?.[0].phase, 'superseded'); +}); + +class SnapshotGapPorts extends InMemoryGoalSessionPorts { + snapshotPersisted: (() => void) | undefined; + + override async compareAndSet(expected: GoalSessionState, next: Omit) { + const saved = await super.compareAndSet(expected, next); + if (saved && expected.providerOpenAttemptId !== next.providerOpenAttemptId + && next.controllerEpoch === 2) this.snapshotPersisted?.(); + return saved; + } +} + +test('reopen never plain-CASes snapshot model ahead of unresolved intent atomic audit', async () => { + const effects: Effects = { model: 'model-a', calls: [] }; + const ports = new SnapshotGapPorts(); + const initial = await opened(new ExactHeadAdapter(effects), ports); + ports.setTransitionFault('before_commit'); + await assert.rejects(initial.supervisor.requestModelChange({ ...control, model: 'model-b' }), /before state\/audit/); + assert.equal((await ports.load(identity))?.currentModel, 'model-a'); + assert.equal(effects.model, 'model-b'); + + const retryGate = deferred(); + const retryStarted = deferred(); + const replacementAdapter = new ExactHeadAdapter(effects); + replacementAdapter.modelGates.set('model-b', retryGate.promise); + replacementAdapter.modelStarted.set('model-b', retryStarted.resolve); + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + const reopening = replacement.openSession({ ...identity, provider: replacementAdapter.provider, controllerEpoch: 2 }); + await retryStarted.promise; + assert.equal((await ports.load(identity))?.currentModel, 'model-a'); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'model_changed').length, 0); + retryGate.resolve(); + const reopened = await reopening; + assert.equal(reopened.currentModel, 'model-b'); + const changes = (await ports.replay(identity)).filter(record => record.event.type === 'model_changed'); + assert.equal(changes.length, 1); + assert.deepEqual(changes[0].event, { + type: 'model_changed', previousModel: 'model-a', model: 'model-b', + }); + await replacement.openSession({ ...identity, provider: replacementAdapter.provider, controllerEpoch: 3 }); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); +}); + +class RecoveryClaimHookPorts extends InMemoryGoalSessionPorts { + afterClaim: (() => Promise) | undefined; + + override async compareAndSet(expected: GoalSessionState, next: Omit) { + const saved = await super.compareAndSet(expected, next); + if (saved && next.recoveryAttempt?.phase === 'claimed' && this.afterClaim) { + const hook = this.afterClaim; + this.afterClaim = undefined; + await hook(); + } + return saved; + } +} + +async function recoverableRuntime(adapter: ExactHeadAdapter, ports: InMemoryGoalSessionPorts) { + adapter.stream = async function* () { + yield { type: 'pause_boundary', boundary: 'recoverable', providerEventId: 'recoverable-boundary' }; + }; + const runtime = await opened(adapter, ports); + await runtime.supervisor.runTurn(turnRequest()); + ports.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: repository.repository, + observedBranch: repository.branch, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + resolvedWorktreePath: repository.worktreePath, + }); + ports.setContainerInspection(identity, { + status: 'running', + recoveryIdentity: { + ...identity, + executionEpoch: 1, + turnId: fence.turnId, + attemptId: 'exact-attempt', + worktreeFingerprint: fingerprintGoalWorktree(repository), + }, + }); + return runtime; +} + +test('cancellation preempts a claimed recovery before any provider resume call', async () => { + const adapter = new ExactHeadAdapter(); + const ports = new RecoveryClaimHookPorts(); + const { supervisor } = await recoverableRuntime(adapter, ports); + ports.afterClaim = async () => { + await supervisor.cancel({ ...control, reason: 'cancel before provider resume' }); + }; + const result = await supervisor.reconcile(identity, 1, repository); + assert.equal(result.state.status, 'terminated'); + assert.equal(adapter.reconcileCalls, 0); + assert.equal(adapter.cancelCalls, 1); + assert.equal((await supervisor.reconcile(identity, 1, repository)).state.status, 'terminated'); +}); + +test('cancellation durably preempts an in-doubt reconciliation without waiting for its provider call', async () => { + const adapter = new ExactHeadAdapter(); + const ports = new InMemoryGoalSessionPorts(); + const { supervisor } = await recoverableRuntime(adapter, ports); + const started = deferred(); + const release = deferred(); + adapter.reconcileStarted = started.resolve; + adapter.reconcileGate = release.promise; + const reconciling = supervisor.reconcile(identity, 1, repository); + await started.promise; + const cancelled = await supervisor.cancel({ ...control, reason: 'cancel during recovery' }); + assert.equal(cancelled.status, 'terminated'); + assert.ok((await ports.load(identity))?.cancellationIntent); + release.resolve(); + await assert.rejects(reconciling, StaleGoalSessionFenceError); + assert.equal(adapter.reconcileCalls, 1); + assert.equal(adapter.cancelCalls, 1); + assert.equal((await supervisor.reconcile(identity, 1, repository)).state.status, 'terminated'); + assert.equal(adapter.reconcileCalls, 1); +}); + +test('replacement cancellation recovers an old recovery lease without post-claim provider resume', async () => { + const adapter = new ExactHeadAdapter(); + const ports = new InMemoryGoalSessionPorts(); + const { supervisor } = await recoverableRuntime(adapter, ports); + const started = deferred(); + const release = deferred(); + adapter.reconcileStarted = started.resolve; + adapter.reconcileGate = release.promise; + const oldRecovery = supervisor.reconcile(identity, 1, repository); + await started.promise; + + const replacement = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await replacement.takeover(identity, 2); + const cancellation = await replacement.cancel({ + ...identity, controllerEpoch: 2, reason: 'replacement cancel', + }); + assert.equal(cancellation.status, 'terminated'); + release.resolve(); + await assert.rejects(oldRecovery, StaleGoalSessionFenceError); + assert.equal(adapter.reconcileCalls, 1); + assert.equal(adapter.cancelCalls, 1); + assert.equal((await replacement.reconcile(identity, 2, repository)).state.status, 'terminated'); +}); + +test('same-controller cancellation can recover a completed failed reconciliation lease', async () => { + const adapter = new ExactHeadAdapter(); + adapter.reconcileFailure = true; + const ports = new InMemoryGoalSessionPorts(); + const { supervisor } = await recoverableRuntime(adapter, ports); + await assert.rejects(supervisor.reconcile(identity, 1, repository), /Provider operation failed safely/); + assert.equal((await ports.load(identity))?.recoveryAttempt?.phase, 'provider_in_doubt'); + assert.equal((await supervisor.cancel({ ...control, reason: 'cancel failed recovery' })).status, 'terminated'); + assert.equal(adapter.reconcileCalls, 1); + assert.equal(adapter.cancelCalls, 1); +}); + +class DurableGapPorts extends SqliteGoalSessionTestPorts { + readonly reached = deferred(); + readonly release = deferred(); + + constructor( + filename: string, + private readonly gap: 'container' | 'repository' | 'claimed' | 'provider_in_doubt', + ) { super(filename); } + + override async inspectContainer(value: typeof identity) { + const inspection = await super.inspectContainer(value); + if (this.gap === 'container') { + this.reached.resolve(); + await this.release.promise; + } + return inspection; + } + + override async inspectRepository(value: typeof repository) { + const inspection = await super.inspectRepository(value); + if (this.gap === 'repository') { + this.reached.resolve(); + await this.release.promise; + } + return inspection; + } + + override async compareAndSet(expected: GoalSessionState, next: Omit) { + const saved = await super.compareAndSet(expected, next); + if (saved && next.recoveryAttempt?.phase === this.gap) { + this.reached.resolve(); + await this.release.promise; + } + return saved; + } +} + +async function recoverableSqliteRuntime(adapter: ExactHeadAdapter, ports: SqliteGoalSessionTestPorts) { + adapter.stream = async function* () { + yield { type: 'pause_boundary', boundary: 'recoverable', providerEventId: 'recoverable-boundary' }; + }; + const runtime = await opened(adapter, ports); + await runtime.supervisor.runTurn(turnRequest()); + ports.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: repository.repository, + observedBranch: repository.branch, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + resolvedWorktreePath: repository.worktreePath, + }); + ports.setContainerInspection(identity, { + status: 'running', + recoveryIdentity: { + ...identity, + executionEpoch: 1, + turnId: fence.turnId, + attemptId: 'exact-attempt', + worktreeFingerprint: fingerprintGoalWorktree(repository), + }, + }); + return runtime; +} + +class RecoveryCommitGapPorts extends SqliteGoalSessionTestPorts { + readonly reached = deferred(); + readonly release = deferred(); + + override async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ) { + if (!('scope' in operation) + && operation.auditEvents.some(event => event.type === 'reconciliation')) { + this.reached.resolve(); + await this.release.promise; + } + return super.commit(expected, next, operation); + } +} + +test('separate-process cancellation fences every inspection and recovery-promotion gap', async t => { + for (const gap of ['container', 'repository', 'claimed', 'provider_in_doubt'] as const) { + await t.test(gap, async subtest => { + const persistence = sqlitePersistence(); + const seedPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const adapter = new ExactHeadAdapter(); + await recoverableSqliteRuntime(adapter, seedPorts); + const recoveryPorts = new DurableGapPorts(persistence.filename, gap); + const cancellationPorts = new SqliteGoalSessionTestPorts(persistence.filename); + subtest.after(() => { + seedPorts.close(); recoveryPorts.close(); cancellationPorts.close(); persistence.cleanup(); + }); + const recovering = new GoalSessionSupervisor(adapter, recoveryPorts.asRuntimePorts()); + const cancelling = new GoalSessionSupervisor(adapter, cancellationPorts.asRuntimePorts()); + const recovery = recovering.reconcile(identity, 1, repository); + await recoveryPorts.reached.promise; + assert.equal((await cancelling.cancel({ ...control, reason: `cancel at ${gap}` })).status, 'terminated'); + recoveryPorts.release.resolve(); + if (gap === 'provider_in_doubt') await assert.rejects(recovery, StaleGoalSessionFenceError); + else assert.equal((await recovery).state.status, 'terminated'); + assert.equal(adapter.reconcileCalls, 0); + assert.equal((await cancellationPorts.replay(identity)).at(-1)?.event.type, 'completion'); + }); + } +}); + +test('SQLite-backed replacement cancellation never waits for a hung provider recovery call', async t => { + const persistence = sqlitePersistence(); + const seedPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const recoveryPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const cancellationPorts = new SqliteGoalSessionTestPorts(persistence.filename); + t.after(() => { seedPorts.close(); recoveryPorts.close(); cancellationPorts.close(); persistence.cleanup(); }); + const adapter = new ExactHeadAdapter(); + await recoverableSqliteRuntime(adapter, seedPorts); + const started = deferred(); + const release = deferred(); + adapter.reconcileStarted = started.resolve; + adapter.reconcileGate = release.promise; + const recovering = new GoalSessionSupervisor(adapter, recoveryPorts.asRuntimePorts()); + const cancelling = new GoalSessionSupervisor(adapter, cancellationPorts.asRuntimePorts()); + const recovery = recovering.reconcile(identity, 1, repository); + await started.promise; + assert.equal((await cancelling.cancel({ ...control, reason: 'cancel hung cross-process recovery' })).status, 'terminated'); + assert.equal(adapter.cancelCalls, 1); + release.resolve(); + await assert.rejects(recovery, StaleGoalSessionFenceError); + assert.equal((await cancellationPorts.load(identity))?.status, 'terminated'); +}); + +test('atomic recovery result and audit replay exactly once across pre/post-commit crashes', async t => { + for (const fault of ['before_commit', 'after_commit'] as const) { + await t.test(fault, async subtest => { + const persistence = sqlitePersistence(); + const firstPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const adapter = new ExactHeadAdapter(); + const { supervisor } = await recoverableSqliteRuntime(adapter, firstPorts); + firstPorts.setTransitionFault(fault); + await assert.rejects(supervisor.reconcile(identity, 1, repository), /Injected crash/); + const replacementPorts = new SqliteGoalSessionTestPorts(persistence.filename); + subtest.after(() => { firstPorts.close(); replacementPorts.close(); persistence.cleanup(); }); + const replacementAdapter = new ExactHeadAdapter(adapter.effects); + const replacement = new GoalSessionSupervisor(replacementAdapter, replacementPorts.asRuntimePorts()); + const recovered = await replacement.reconcile(identity, 1, repository); + assert.equal(recovered.outcome, 'resumed'); + const audits = (await replacementPorts.replay(identity)) + .filter(record => record.event.type === 'reconciliation'); + assert.equal(audits.length, 1); + assert.equal(adapter.reconcileCalls + replacementAdapter.reconcileCalls, + fault === 'before_commit' ? 2 : 1); + }); + } +}); + +test('cancellation at the recovery state-to-audit gap wins one atomic terminal ordering', async t => { + const persistence = sqlitePersistence(); + const seedPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const adapter = new ExactHeadAdapter(); + await recoverableSqliteRuntime(adapter, seedPorts); + const recoveryPorts = new RecoveryCommitGapPorts(persistence.filename); + const cancellationPorts = new SqliteGoalSessionTestPorts(persistence.filename); + t.after(() => { seedPorts.close(); recoveryPorts.close(); cancellationPorts.close(); persistence.cleanup(); }); + const recovering = new GoalSessionSupervisor(adapter, recoveryPorts.asRuntimePorts()); + const cancelling = new GoalSessionSupervisor(adapter, cancellationPorts.asRuntimePorts()); + const recovery = recovering.reconcile(identity, 1, repository); + await recoveryPorts.reached.promise; + assert.equal((await cancelling.cancel({ ...control, reason: 'cancel before recovery transaction' })).status, 'terminated'); + recoveryPorts.release.resolve(); + await assert.rejects(recovery, StaleGoalSessionFenceError); + const events = await cancellationPorts.replay(identity); + assert.equal(events.filter(record => record.event.type === 'reconciliation').length, 0); + assert.equal(events.at(-1)?.event.type, 'completion'); +}); + +test('an expired preparing lease is reclaimed through SQLite and the crashed owner stays fenced', async t => { + const persistence = sqlitePersistence(); + const seedPorts = new SqliteGoalSessionTestPorts(persistence.filename); + const oldAdapter = new ExactHeadAdapter(); + await recoverableSqliteRuntime(oldAdapter, seedPorts); + const crashedPorts = new DurableGapPorts(persistence.filename, 'claimed'); + const crashed = new GoalSessionSupervisor(oldAdapter, crashedPorts.asRuntimePorts()); + const oldRecovery = crashed.reconcile(identity, 1, repository); + await crashedPorts.reached.promise; + + const replacementPorts = new SqliteGoalSessionTestPorts(persistence.filename); + t.after(() => { + seedPorts.close(); crashedPorts.close(); replacementPorts.close(); persistence.cleanup(); + }); + const claimed = await replacementPorts.load(identity); + assert.ok(claimed?.recoveryAttempt); + const expired = await replacementPorts.compareAndSet(claimed, { + ...claimed, + recoveryAttempt: { ...claimed.recoveryAttempt, leaseExpiresAt: new Date(0).toISOString() }, + }); + assert.ok(expired); + const replacementAdapter = new ExactHeadAdapter(oldAdapter.effects); + const replacement = new GoalSessionSupervisor(replacementAdapter, replacementPorts.asRuntimePorts()); + assert.equal((await replacement.reconcile(identity, 1, repository)).outcome, 'resumed'); + crashedPorts.release.resolve(); + await assert.rejects(oldRecovery, StaleGoalSessionFenceError); + assert.equal(oldAdapter.reconcileCalls, 0); + assert.equal(replacementAdapter.reconcileCalls, 1); +}); diff --git a/packages/core/test/goalSessionFinalReaudit.test.ts b/packages/core/test/goalSessionFinalReaudit.test.ts new file mode 100644 index 000000000..d76830ad0 --- /dev/null +++ b/packages/core/test/goalSessionFinalReaudit.test.ts @@ -0,0 +1,397 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalProviderCancelRequest, + GoalProviderCapabilities, + GoalProviderModelChangeRequest, + GoalProviderOpenRequest, + GoalProviderReconcileRequest, + GoalSessionAdapter, + GoalSessionControlFence, + GoalSessionControlTransition, + GoalSessionEvent, + GoalSessionState, + GoalTerminalCommit, +} from '../src/agents/goalSession/contract.js'; +import { + GoalSessionSupervisor, + StaleGoalSessionFenceError, +} from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; + +const identity = { goalId: 'final-reaudit-goal', sessionId: 'final-reaudit-session' }; +const control = { ...identity, controllerEpoch: 1 }; +const fence = { ...control, turnId: 'final-reaudit-turn' }; +const repository = { + repository: 'integry/propr', worktreePath: '/tmp/final-reaudit', branch: 'reaudit', headSha: '10cea2fc', +}; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +type SharedEffects = { modelIds: Set; cancelIds: Set; model: string }; + +class FinalReauditAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'final-reaudit-provider'; + readonly modelCalls: GoalProviderModelChangeRequest[] = []; + readonly cancelCalls: GoalProviderCancelRequest[] = []; + readonly capabilities: GoalProviderCapabilities; + openCalls = 0; + reconcileCalls = 0; + pauseCalls = 0; + pauseAcknowledgement = { appliesAt: 'next_safe_boundary' as const }; + pauseStarted: (() => void) | undefined; + pauseGate: Promise | undefined; + cancelStarted: (() => void) | undefined; + cancelGate: Promise | undefined; + stream: (request: GoalBeginTurnRequest) => AsyncIterable = async function* () { + yield { type: 'completion', outcome: 'succeeded' }; + }; + + constructor( + capabilities: GoalProviderCapabilities = { + nativeSessionId: 'eager', steering: 'active_turn', pause: 'active_turn', modelChange: 'next_safe_boundary', + }, + readonly effects: SharedEffects = { modelIds: new Set(), cancelIds: new Set(), model: 'model-a' }, + ) { this.capabilities = capabilities; } + + async openSession(_request: GoalProviderOpenRequest) { + this.openCalls += 1; + return { providerSessionId: 'final-native', recoveryMetadata: { checkpoint: 'open' }, model: this.effects.model }; + } + + beginTurn(request: GoalBeginTurnRequest): AsyncIterable { return this.stream(request); } + + async resumeSession(_request: GoalSessionControlFence, snapshot: { providerSessionId: string; recoveryMetadata: unknown }) { + return snapshot; + } + + async requestPause() { + this.pauseCalls += 1; + this.pauseStarted?.(); + if (this.pauseGate) await this.pauseGate; + return this.pauseAcknowledgement; + } + + async requestModelChange(request: GoalProviderModelChangeRequest) { + this.modelCalls.push(structuredClone(request)); + this.effects.modelIds.add(request.modelChangeId); + this.effects.model = request.model; + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + + async cancel(request: GoalProviderCancelRequest): Promise { await this.signalCancel(request); } + + async cancelPending(request: GoalProviderCancelRequest): Promise { await this.signalCancel(request); } + + async reconcile(_request: GoalProviderReconcileRequest) { + this.reconcileCalls += 1; + return { outcome: 'resumed' as const, snapshot: await this.openSession({} as GoalProviderOpenRequest), reason: 'resumed' }; + } + + private async signalCancel(request: GoalProviderCancelRequest): Promise { + this.cancelCalls.push(structuredClone(request)); + this.effects.cancelIds.add(request.cancellationId); + this.cancelStarted?.(); + if (this.cancelGate) await this.cancelGate; + } +} + +async function openRuntime(adapter: FinalReauditAdapter, ports = new InMemoryGoalSessionPorts()) { + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + return { ports, supervisor }; +} + +function turnRequest() { + return { + ...fence, executionId: 'final-execution', attemptId: 'final-attempt', objective: 'adversarial re-audit', + repository, requestedModel: 'model-a', + }; +} + +async function startHeldTurn(adapter: FinalReauditAdapter, supervisor: GoalSessionSupervisor) { + const started = deferred(); + const release = deferred(); + adapter.stream = async function* () { + started.resolve(); + await release.promise; + yield { type: 'completion', outcome: 'succeeded' }; + }; + const running = supervisor.runTurn(turnRequest()); + await started.promise; + return { release, running }; +} + +test('eager pause request and acknowledgement boundaries commit state plus audit atomically', async t => { + for (const fault of ['before_commit', 'after_commit'] as const) { + await t.test(`request ${fault}`, async () => { + const adapter = new FinalReauditAdapter(); + const { ports, supervisor } = await openRuntime(adapter); + const turn = await startHeldTurn(adapter, supervisor); + ports.setTransitionFault(fault); + await assert.rejects(supervisor.requestPause({ ...control, reason: 'pause' }), /transaction commit/); + const committed = fault === 'after_commit'; + assert.equal((await ports.load(identity))?.status, committed ? 'pause_requested' : 'running'); + assert.equal((await ports.replay(identity)).filter(value => value.event.type === 'pause_requested').length, + committed ? 1 : 0); + await supervisor.requestPause({ ...control, reason: 'retry same pause' }); + assert.equal((await ports.replay(identity)).filter(value => value.event.type === 'pause_requested').length, 1); + await supervisor.cancel({ ...control, reason: 'finish' }); + turn.release.resolve(); + await assert.rejects(turn.running, StaleGoalSessionFenceError); + }); + } + + for (const fault of ['before_commit', 'after_commit'] as const) { + await t.test(`boundary ${fault}`, async () => { + const adapter = new FinalReauditAdapter(); + const { ports, supervisor } = await openRuntime(adapter); + const turn = await startHeldTurn(adapter, supervisor); + await supervisor.requestPause({ ...control }); + adapter.pauseAcknowledgement = { + appliesAt: 'next_safe_boundary', boundaryReached: { boundary: 'provider-safe' }, + }; + ports.setTransitionFault(fault); + await assert.rejects(supervisor.requestPause({ ...control }), /transaction commit/); + if (fault === 'before_commit') await supervisor.requestPause({ ...control }); + const audits = (await ports.replay(identity)).filter(value => + value.event.type === 'pause_requested' || value.event.type === 'pause_boundary'); + assert.deepEqual(audits.map(value => value.event.type), ['pause_requested', 'pause_boundary']); + assert.equal((await ports.load(identity))?.status, 'paused'); + await supervisor.cancel({ ...control, reason: 'finish' }); + turn.release.resolve(); + await assert.rejects(turn.running, StaleGoalSessionFenceError); + }); + } +}); + +class BeforeTransitionPorts extends InMemoryGoalSessionPorts { + beforeTransition: ((operation: GoalSessionControlTransition) => Promise) | undefined; + + override async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ) { + if (!('scope' in operation) && this.beforeTransition) { + const hook = this.beforeTransition; + this.beforeTransition = undefined; + await hook(operation); + } + return super.commit(expected, next, operation); + } +} + +test('eager pause and streamed model/pause transitions lose atomically to cancellation', async t => { + for (const kind of ['eager_pause', 'model_changed', 'pause_boundary'] as const) { + await t.test(kind, async () => { + const adapter = new FinalReauditAdapter(); + const ports = new BeforeTransitionPorts(); + const { supervisor } = await openRuntime(adapter, ports); + let pending: Promise; + let release: (() => void) | undefined; + if (kind === 'eager_pause') { + const held = await startHeldTurn(adapter, supervisor); + release = held.release.resolve; + void held.running.catch(() => undefined); + pending = supervisor.requestPause({ ...control }); + } else { + adapter.stream = async function* () { + yield kind === 'model_changed' + ? { + type: 'model_changed', previousModel: 'model-a', model: 'model-b', providerEventId: 'stream-model-1', + } + : { type: 'pause_boundary', boundary: 'provider-safe', providerEventId: 'stream-pause-1' }; + }; + pending = supervisor.runTurn(turnRequest()); + } + ports.beforeTransition = async operation => { + if (operation.auditEvents.some(event => event.type === kind || kind === 'eager_pause')) { + await supervisor.cancel({ ...control, reason: 'cancellation wins' }); + } + }; + await assert.rejects(pending, StaleGoalSessionFenceError); + release?.(); + const events = await ports.replay(identity); + assert.deepEqual(events.map(value => value.event.type), ['completion']); + assert.equal((await ports.load(identity))?.status, 'terminated'); + }); + } +}); + +test('streamed model and pause events survive transition crash windows without split or duplicate audit', async t => { + for (const eventType of ['model_changed', 'pause_boundary'] as const) { + for (const fault of ['before_commit', 'after_commit'] as const) { + await t.test(`${eventType} ${fault}`, async () => { + const adapter = new FinalReauditAdapter(); + adapter.stream = async function* () { + yield eventType === 'model_changed' + ? { + type: 'model_changed', previousModel: 'model-a', model: 'model-b', providerEventId: 'stream-model-2', + } + : { type: 'pause_boundary', boundary: 'provider-safe', providerEventId: 'stream-pause-2' }; + }; + const { ports, supervisor } = await openRuntime(adapter); + ports.setTransitionFault(fault); + await assert.rejects(supervisor.runTurn(turnRequest()), /transaction commit/); + const events = await ports.replay(identity); + assert.equal(events.filter(value => value.event.type === eventType).length, + fault === 'after_commit' ? 1 : 0); + assert.equal(events.filter(value => value.event.type === 'completion').length, 0, + 'a runtime persistence crash leaves the exact invocation recoverable'); + assert.equal((await ports.load(identity))?.activeTurn?.attemptId, 'final-attempt'); + }); + } + } +}); + +class CrashBeforeModelCallPorts extends InMemoryGoalSessionPorts { + private crash = true; + + override async compareAndSet(expected: GoalSessionState, next: Omit) { + const saved = await super.compareAndSet(expected, next); + if (this.crash && expected.modelChangeIntent?.phase !== 'provider_in_doubt' + && next.modelChangeIntent?.phase === 'provider_in_doubt') { + this.crash = false; + throw new Error('Injected crash after in-doubt phase before provider call'); + } + return saved; + } +} + +test('next-safe-boundary model intent reuses one provider identity across every crash/reopen window', async t => { + for (const fault of ['pre_call', 'pre_commit', 'post_commit'] as const) { + await t.test(fault, async () => { + const effects: SharedEffects = { modelIds: new Set(), cancelIds: new Set(), model: 'model-a' }; + const initialAdapter = new FinalReauditAdapter(undefined, effects); + const ports = fault === 'pre_call' ? new CrashBeforeModelCallPorts() : new InMemoryGoalSessionPorts(); + const { supervisor } = await openRuntime(initialAdapter, ports); + if (fault === 'pre_commit') ports.setTransitionFault('before_commit'); + if (fault === 'post_commit') ports.setTransitionFault('after_commit'); + await assert.rejects(supervisor.requestModelChange({ ...control, model: 'model-b' }), /Injected crash/); + const intent = (await ports.load(identity))?.modelChangeIntent; + assert.ok(intent); + assert.equal(intent.phase, fault === 'post_commit' ? 'committed' : 'provider_in_doubt'); + + const replacementAdapter = new FinalReauditAdapter(undefined, effects); + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + const reopened = await replacement.openSession({ + ...identity, provider: replacementAdapter.provider, controllerEpoch: 2, + }); + const calls = [...initialAdapter.modelCalls, ...replacementAdapter.modelCalls]; + assert.equal(new Set(calls.map(call => call.modelChangeId)).size, calls.length ? 1 : 0); + assert.equal(effects.modelIds.size, 1); + assert.equal(reopened.modelChangeIntent?.modelChangeId, intent.modelChangeId); + assert.equal(reopened.modelChangeIntent?.phase, 'committed'); + assert.equal(reopened.currentModel, 'model-b'); + assert.equal((await ports.replay(identity)).filter(value => + value.event.type === 'model_change_acknowledged').length, 1); + assert.equal(replacementAdapter.modelCalls.length, fault === 'post_commit' ? 0 : 1); + }); + } +}); + +test('reconcile routes bound and unbound cancelling sessions only through stable cancellation recovery', async t => { + for (const binding of ['bound', 'unbound'] as const) { + await t.test(binding, async () => { + const capabilities: GoalProviderCapabilities = binding === 'bound' + ? { nativeSessionId: 'eager', steering: 'active_turn', pause: 'active_turn', modelChange: 'next_safe_boundary' } + : { + nativeSessionId: 'first_turn', firstTurnIdCrashPolicy: 'retry_deterministically', + steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + }; + const effects: SharedEffects = { modelIds: new Set(), cancelIds: new Set(), model: 'model-a' }; + const initialAdapter = new FinalReauditAdapter(capabilities, effects); + const initialStarted = deferred(); + const initialRelease = deferred(); + initialAdapter.cancelStarted = initialStarted.resolve; + initialAdapter.cancelGate = initialRelease.promise; + const { ports, supervisor } = await openRuntime(initialAdapter); + const cancelling = supervisor.cancel({ ...control, reason: 'crash during cancellation' }); + await initialStarted.promise; + + const replacementAdapter = new FinalReauditAdapter(capabilities, effects); + const replacementStarted = deferred(); + const replacementRelease = deferred(); + replacementAdapter.cancelStarted = replacementStarted.resolve; + replacementAdapter.cancelGate = replacementRelease.promise; + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + const recovery = replacement.reconcile(identity, 2, repository); + await replacementStarted.promise; + assert.equal(replacementAdapter.reconcileCalls, 0); + assert.equal(replacementAdapter.openCalls, 0); + assert.equal(new Set([ + initialAdapter.cancelCalls[0].cancellationId, + replacementAdapter.cancelCalls[0].cancellationId, + ]).size, 1); + initialRelease.resolve(); + assert.equal((await cancelling).status, 'terminated'); + replacementRelease.resolve(); + assert.equal((await recovery).state.status, 'terminated'); + assert.equal((await replacement.reconcile(identity, 2, repository)).state.status, 'terminated'); + assert.equal(replacementAdapter.reconcileCalls, 0); + }); + } +}); + +test('terminated and failed sessions never inspect/provider-reconcile or resurrect on repeat replacement', async t => { + for (const terminal of ['terminated', 'failed'] as const) { + await t.test(terminal, async () => { + const adapter = new FinalReauditAdapter(); + const { ports, supervisor } = await openRuntime(adapter); + if (terminal === 'terminated') await supervisor.cancel({ ...control, reason: 'terminal' }); + else { + adapter.stream = async function* () { yield { type: 'completion', outcome: 'failed', error: 'failed' }; }; + await supervisor.runTurn(turnRequest()); + } + const replacementAdapter = new FinalReauditAdapter(); + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = await replacement.reconcile(identity, 2, repository); + assert.equal(result.outcome, 'blocked'); + assert.equal(result.state.status, terminal); + } + assert.equal(replacementAdapter.reconcileCalls, 0); + assert.equal(replacementAdapter.openCalls, 0); + }); + } +}); + +class CancelCompletionRacePorts extends InMemoryGoalSessionPorts { + beforeTakeover: (() => Promise) | undefined; + + override async compareAndSet(expected: GoalSessionState, next: Omit) { + if (next.controllerEpoch > expected.controllerEpoch && this.beforeTakeover) { + const hook = this.beforeTakeover; + this.beforeTakeover = undefined; + await hook(); + } + return super.compareAndSet(expected, next); + } +} + +test('reconcile cannot invalidate cancellation completion racing its durable takeover claim', async () => { + const adapter = new FinalReauditAdapter(); + const started = deferred(); + const release = deferred(); + adapter.cancelStarted = started.resolve; + adapter.cancelGate = release.promise; + const ports = new CancelCompletionRacePorts(); + const { supervisor } = await openRuntime(adapter, ports); + const cancelling = supervisor.cancel({ ...control, reason: 'finish during reconcile takeover' }); + await started.promise; + const replacementAdapter = new FinalReauditAdapter(); + replacementAdapter.cancelStarted = release.resolve; + const replacement = new GoalSessionSupervisor(replacementAdapter, ports.asRuntimePorts()); + const result = await replacement.reconcile(identity, 2, repository); + assert.equal(result.state.status, 'terminated'); + assert.equal((await cancelling).status, 'terminated'); + assert.equal(replacementAdapter.reconcileCalls, 0); + assert.equal(replacementAdapter.cancelCalls.length, 1); +}); diff --git a/packages/core/test/goalSessionOwnerAddendum.test.ts b/packages/core/test/goalSessionOwnerAddendum.test.ts new file mode 100644 index 000000000..3c0a7fbc5 --- /dev/null +++ b/packages/core/test/goalSessionOwnerAddendum.test.ts @@ -0,0 +1,647 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalCancelRequest, + GoalModelChangeRequest, + GoalProviderOpenRequest, + GoalProviderReconcileRequest, + GoalProviderReconcileResult, + GoalProviderCapabilities, + GoalProviderSessionSnapshot, + GoalSessionAdapter, + GoalSessionControlFence, + GoalSessionEvent, + GoalSessionFence, + GoalSessionState, +} from '../src/agents/goalSession/contract.js'; +import { + GoalSessionSupervisor, + StaleGoalSessionFenceError, +} from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; + +const identity = { goalId: 'owner-goal', sessionId: 'owner-session' }; +const repository = { + repository: 'integry/propr', + worktreePath: '/tmp/owner-goal-worktree', + branch: 'owner-branch', + headSha: 'starting-head', +}; +const fence: GoalSessionFence = { ...identity, controllerEpoch: 1, turnId: 'owner-turn' }; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +class AddendumAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'owner-test'; + readonly capabilities: GoalProviderCapabilities = { + nativeSessionId: 'eager', + steering: 'active_turn', + pause: 'active_turn', + modelChange: 'next_safe_boundary', + } as const; + turn: (request: GoalBeginTurnRequest) => AsyncIterable = async function* () { + yield { type: 'completion', outcome: 'succeeded' }; + }; + resumedTurn: () => AsyncIterable = async function* () { + yield { type: 'completion', outcome: 'succeeded' }; + }; + cancelTurn: (_request: GoalCancelRequest) => Promise = async () => undefined; + reconcileTurn: (_request: GoalProviderReconcileRequest) => Promise + = async () => ({ outcome: 'alive', reason: 'alive' }); + reconcileRequests: GoalProviderReconcileRequest[] = []; + + async openSession(_request: GoalProviderOpenRequest): Promise { + return { providerSessionId: 'owner-provider-session', recoveryMetadata: { checkpoint: 'opened' }, model: 'model-a' }; + } + + beginTurn(request: GoalBeginTurnRequest): AsyncIterable { + return this.turn(request); + } + + resumeTurn(): AsyncIterable { + return this.resumedTurn(); + } + + async resumeSession( + _request: GoalSessionControlFence, + snapshot: GoalProviderSessionSnapshot, + ): Promise { + return snapshot; + } + + async requestModelChange(request: GoalModelChangeRequest) { + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + + cancel(request: GoalCancelRequest): Promise { + return this.cancelTurn(request); + } + + async reconcile(request: GoalProviderReconcileRequest): Promise { + this.reconcileRequests.push(structuredClone(request)); + return this.reconcileTurn(request); + } +} + +async function openRuntime(adapter: AddendumAdapter, ids?: string[]) { + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), ids ? () => ids.shift()! : undefined); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + return { persistence, supervisor }; +} + +test('durable cancellation prevents a completion racing provider cancellation from resurrecting the session', async () => { + const adapter = new AddendumAdapter(); + const turnStarted = deferred(); + const releaseCompletion = deferred(); + const cancelStarted = deferred(); + const releaseCancel = deferred(); + adapter.turn = async function* () { + turnStarted.resolve(); + await releaseCompletion.promise; + yield { type: 'completion', outcome: 'succeeded' }; + }; + adapter.cancelTurn = async () => { + cancelStarted.resolve(); + await releaseCancel.promise; + }; + const { persistence, supervisor } = await openRuntime(adapter); + const running = supervisor.runTurn({ + ...fence, + executionId: 'execution-cancel-race', + attemptId: 'attempt-cancel-race', + objective: 'race cancellation', + repository, + requestedModel: 'model-a', + }); + await turnStarted.promise; + const cancelling = supervisor.cancel({ ...fence, reason: 'owner cancelled' }); + await cancelStarted.promise; + assert.equal((await persistence.load(identity))?.status, 'cancelling'); + + releaseCompletion.resolve(); + await assert.rejects(running, StaleGoalSessionFenceError); + assert.equal((await persistence.load(identity))?.status, 'cancelling'); + + releaseCancel.resolve(); + const terminal = await cancelling; + assert.equal(terminal.status, 'terminated'); + assert.equal(terminal.activeTurn, undefined); + const completions = (await persistence.replay(identity)).filter(record => record.event.type === 'completion'); + assert.equal(completions.length, 1); + assert.equal(completions[0].event.type === 'completion' ? completions[0].event.outcome : '', 'cancelled'); +}); + +test('actual stale streams cannot mutate checkpoint, model, or pause after recovery starts a fresh attempt', async t => { + const cases: Array<{ + name: string; + event: GoalSessionEvent; + verify: (state: GoalSessionState) => void; + }> = [ + { + name: 'checkpoint recovery metadata', + event: { type: 'checkpoint', checkpointId: 'stale', recoveryMetadata: { checkpoint: 'stale' } }, + verify: state => assert.deepEqual(state.recoveryMetadata, { checkpoint: 'recovered' }), + }, + { + name: 'current model', + event: { + type: 'model_changed', previousModel: 'model-a', model: 'stale-model', providerEventId: 'stale-model-event', + }, + verify: state => assert.equal(state.currentModel, 'model-recovered'), + }, + { + name: 'pause boundary', + event: { type: 'pause_boundary', boundary: 'stale-boundary', providerEventId: 'stale-pause-event' }, + verify: state => { + assert.equal(state.status, 'running'); + assert.equal(state.activeTurn?.status, 'running'); + }, + }, + ]; + + for (const testCase of cases) { + await t.test(testCase.name, async () => { + const adapter = new AddendumAdapter(); + const oldTurnStarted = deferred(); + const releaseOldEvent = deferred(); + const currentTurnStarted = deferred(); + const releaseCurrentTurn = deferred(); + adapter.turn = async function* () { + oldTurnStarted.resolve(); + await releaseOldEvent.promise; + yield testCase.event; + }; + adapter.resumedTurn = async function* () { + currentTurnStarted.resolve(); + await releaseCurrentTurn.promise; + yield { type: 'completion', outcome: 'succeeded' }; + }; + adapter.reconcileTurn = async () => ({ + outcome: 'resumed', + reason: 'fresh recovery attempt enacted', + snapshot: { + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: 'recovered' }, + model: 'model-recovered', + }, + }); + const { persistence, supervisor } = await openRuntime(adapter); + const running = supervisor.runTurn({ + ...fence, + executionId: 'execution-shared', + attemptId: 'attempt-old', + objective: `stale ${testCase.name}`, + repository, + requestedModel: 'model-a', + }); + await oldTurnStarted.promise; + persistence.setContainerInspection(identity, { status: 'missing', reason: 'old invocation disappeared' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: repository.repository, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + resolvedWorktreePath: repository.worktreePath, + }); + const recovered = await supervisor.reconcile(identity, 1, repository); + assert.equal(recovered.outcome, 'resumed'); + const resumed = supervisor.resumeTurn({ ...identity, controllerEpoch: 1 }); + await currentTurnStarted.promise; + const currentAttempt = (await persistence.load(identity))?.activeTurn?.attemptId; + assert.ok(currentAttempt); + assert.notEqual(currentAttempt, 'attempt-old'); + + releaseOldEvent.resolve(); + await assert.rejects(running, StaleGoalSessionFenceError); + + const state = await persistence.load(identity); + assert.ok(state); + assert.equal(state.activeTurn?.attemptId, currentAttempt); + testCase.verify(state); + assert.equal((await persistence.replay(identity)).some(record => + JSON.stringify(record.event) === JSON.stringify(testCase.event)), false); + releaseCurrentTurn.resolve(); + await resumed; + }); + } +}); + +test('an old recovery attempt cannot consume or emit corrective-message acknowledgement', async () => { + class NextTurnMessageAdapter extends AddendumAdapter { + override readonly capabilities: GoalProviderCapabilities = { + nativeSessionId: 'eager', + steering: 'next_turn', + pause: 'active_turn', + modelChange: 'next_safe_boundary', + }; + } + const adapter = new NextTurnMessageAdapter(); + const turnStarted = deferred(); + const releaseOldAcknowledgement = deferred(); + adapter.turn = async function* (request) { + turnStarted.resolve(); + await releaseOldAcknowledgement.promise; + yield { type: 'message_acknowledged', messageId: request.correctiveMessages![0].messageId }; + yield { type: 'completion', outcome: 'succeeded' }; + }; + adapter.reconcileTurn = async () => ({ + outcome: 'resumed', + reason: 'replacement attempt owns the recovered invocation', + snapshot: { + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: 'message-recovery' }, + model: 'model-a', + }, + }); + const { persistence, supervisor } = await openRuntime(adapter, ['attempt-open', 'attempt-recovery']); + persistence.enqueueMessage({ ...identity, messageId: 'message-one', body: 'first correction' }); + persistence.enqueueMessage({ ...identity, messageId: 'message-two', body: 'second correction' }); + const running = supervisor.runTurn({ + ...fence, + executionId: 'execution-message-recovery', + attemptId: 'attempt-old-message', + objective: 'recover before the old acknowledgement arrives', + repository, + requestedModel: 'model-a', + }); + await turnStarted.promise; + persistence.setContainerInspection(identity, { status: 'missing', reason: 'old message attempt disappeared' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: repository.repository, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + resolvedWorktreePath: repository.worktreePath, + }); + const recovered = await supervisor.reconcile(identity, 1, repository); + const currentExecution = { + executionId: recovered.state.activeTurn!.executionId, + attemptId: recovered.state.activeTurn!.attemptId, + }; + assert.equal(currentExecution.attemptId, 'attempt-recovery'); + + releaseOldAcknowledgement.resolve(); + await assert.rejects(running, StaleGoalSessionFenceError); + assert.deepEqual((await persistence.listPending(identity)).map(message => message.messageId), [ + 'message-one', 'message-two', + ]); + assert.equal((await persistence.replay(identity)).some(record => + record.event.type === 'message_acknowledged' && record.attemptId === 'attempt-old-message'), false); + + assert.equal(await persistence.acknowledge(fence, currentExecution, 'message-one'), 'acknowledged'); + assert.equal(await persistence.acknowledge(fence, currentExecution, 'message-one'), 'already_acknowledged'); + const appended = await persistence.append(fence, currentExecution, { + type: 'message_acknowledged', messageId: 'message-one', + }); + assert.equal(appended.accepted, true); + assert.deepEqual((await persistence.listPending(identity)).map(message => message.messageId), ['message-two']); +}); + +async function seededRecovery(adapter: AddendumAdapter, ids: string[]) { + const persistence = new InMemoryGoalSessionPorts(); + const timestamp = new Date().toISOString(); + await persistence.create({ + ...identity, + provider: adapter.provider, + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: 'durable' }, + controllerEpoch: 1, + status: 'running', + currentModel: 'model-a', + requestedModel: 'model-a', + activeTurn: { + executionId: 'execution-live', + attemptId: 'attempt-live', + turnId: fence.turnId, + executionEpoch: 1, + objective: 'recover live turn', + requestedModel: 'model-a', + repository, + status: 'running', + }, + completedTurnIds: [], + createdAt: timestamp, + updatedAt: timestamp, + }); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'container unavailable' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: repository.repository, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + resolvedWorktreePath: repository.worktreePath, + }); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => ids.shift()!); + return { persistence, supervisor }; +} + +class PromotionCrashPorts extends InMemoryGoalSessionPorts { + private crashPromotion = true; + + override async compareAndSet( + expected: GoalSessionState, + next: Omit, + ): Promise { + if (this.crashPromotion && expected.recoveryAttempt + && next.recoveryAttempt === undefined + && next.activeTurn?.attemptId === expected.recoveryAttempt.attemptId) { + this.crashPromotion = false; + throw new Error('Injected crash before replacement promotion'); + } + return super.compareAndSet(expected, next); + } +} + +async function seededRecoveryWithPorts(adapter: AddendumAdapter, ids: string[], persistence: InMemoryGoalSessionPorts) { + const timestamp = new Date().toISOString(); + await persistence.create({ + ...identity, + provider: adapter.provider, + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: 'durable' }, + controllerEpoch: 1, + status: 'running', + currentModel: 'model-a', + requestedModel: 'model-a', + activeTurn: { + executionId: 'execution-live', attemptId: 'attempt-live', turnId: fence.turnId, + executionEpoch: 1, objective: 'recover live turn', requestedModel: 'model-a', + repository, status: 'running', + }, + completedTurnIds: [], + createdAt: timestamp, + updatedAt: timestamp, + }); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'container unavailable' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: repository.repository, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + resolvedWorktreePath: repository.worktreePath, + }); + return { + persistence, + supervisor: new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => ids.shift()!), + }; +} + +test('alive reconciliation preserves the authoritative live attempt while its fresh claim is in flight and after success', async () => { + const adapter = new AddendumAdapter(); + const reconcileStarted = deferred(); + const releaseReconcile = deferred(); + adapter.reconcileTurn = async () => { + reconcileStarted.resolve(); + await releaseReconcile.promise; + return { outcome: 'alive', reason: 'original container remains alive' }; + }; + const { persistence, supervisor } = await seededRecovery(adapter, ['recovery-alive']); + const reconciling = supervisor.reconcile(identity, 2, repository); + await reconcileStarted.promise; + const inFlight = await persistence.load(identity); + assert.equal(inFlight?.activeTurn?.attemptId, 'attempt-live'); + assert.equal(inFlight?.recoveryAttempt?.attemptId, 'recovery-alive'); + + releaseReconcile.resolve(); + const result = await reconciling; + assert.equal(result.outcome, 'alive'); + assert.equal(result.state.activeTurn?.attemptId, 'attempt-live'); + assert.equal(result.state.recoveryAttempt, undefined); + assert.equal((await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'attempt-live', + }, { type: 'output', channel: 'stdout', data: 'still authoritative' })).accepted, true); +}); + +test('a thrown reconciliation preserves live identity and a retry durably claims a fresh attempt', async () => { + const adapter = new AddendumAdapter(); + let call = 0; + adapter.reconcileTurn = async () => { + call += 1; + if (call === 1) throw new Error('reconcile transport failed'); + return { outcome: 'alive', reason: 'retry observed live container' }; + }; + const { persistence, supervisor } = await seededRecovery(adapter, ['recovery-thrown', 'recovery-retry']); + await assert.rejects(supervisor.reconcile(identity, 2, repository), /Provider operation failed safely/); + const failedCall = await persistence.load(identity); + assert.equal(failedCall?.activeTurn?.attemptId, 'attempt-live'); + assert.equal(failedCall?.recoveryAttempt?.attemptId, 'recovery-thrown'); + assert.equal((await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'attempt-live', + }, { type: 'output', channel: 'stdout', data: 'live after thrown reconcile' })).accepted, true); + + const retried = await supervisor.reconcile(identity, 2, repository); + assert.equal(retried.outcome, 'alive'); + assert.equal(retried.state.activeTurn?.attemptId, 'attempt-live'); + assert.deepEqual(adapter.reconcileRequests.map(request => request.attemptId), ['recovery-thrown', 'recovery-retry']); +}); + +test('blocked reconciliation does not claim or replace an attempt', async () => { + const adapter = new AddendumAdapter(); + const { persistence, supervisor } = await seededRecovery(adapter, ['must-not-be-used']); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: 'foreign/repository', + observedBranch: repository.branch, + observedHeadSha: 'foreign-head', + observedWorktreeFingerprint: fingerprintGoalWorktree({ ...repository, repository: 'foreign/repository' }), + }); + const result = await supervisor.reconcile(identity, 2, repository); + assert.equal(result.outcome, 'blocked'); + assert.equal(result.state.activeTurn?.attemptId, 'attempt-live'); + assert.equal((await persistence.load(identity))?.recoveryAttempt, undefined); + assert.equal(adapter.reconcileRequests.length, 0); + assert.equal((await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'attempt-live', + }, { type: 'output', channel: 'stdout', data: 'live after blocked reconcile' })).accepted, true); +}); + +test('replacement reconciliation changes attempt identity only after the adapter proves replacement', async () => { + const adapter = new AddendumAdapter(); + const reconcileStarted = deferred(); + const releaseReconcile = deferred(); + adapter.reconcileTurn = async () => { + reconcileStarted.resolve(); + await releaseReconcile.promise; + return { + outcome: 'resumed', + reason: 'replacement enacted', + snapshot: { + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: 'replacement' }, + model: 'model-a', + }, + }; + }; + const { persistence, supervisor } = await seededRecovery(adapter, ['recovery-replacement']); + const reconciling = supervisor.reconcile(identity, 2, repository); + await reconcileStarted.promise; + assert.equal((await persistence.load(identity))?.activeTurn?.attemptId, 'attempt-live'); + + releaseReconcile.resolve(); + const result = await reconciling; + assert.equal(result.outcome, 'resumed'); + assert.equal(result.state.status, 'paused'); + assert.equal(result.state.activeTurn?.attemptId, 'recovery-replacement'); + assert.equal(result.state.recoveryAttempt, undefined); + assert.deepEqual(await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'attempt-live', + }, { type: 'output', channel: 'stdout', data: 'stale after replacement' }), { + accepted: false, reason: 'turn_not_active', + }); + assert.equal((await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'recovery-replacement', + }, { type: 'output', channel: 'stdout', data: 'replacement output' })).accepted, true); +}); + +test('a crash before replacement promotion preserves old authority and retry promotes only its fresh attempt', async () => { + const adapter = new AddendumAdapter(); + adapter.reconcileTurn = async request => ({ + outcome: 'resumed', + reason: `replacement ${request.attemptId} enacted`, + snapshot: { + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: request.attemptId }, + model: 'model-a', + }, + }); + const persistence = new PromotionCrashPorts(); + const seeded = await seededRecoveryWithPorts( + adapter, + ['recovery-before-crash', 'recovery-after-crash'], + persistence, + ); + persistence.setTransitionFault('before_commit'); + + await assert.rejects(seeded.supervisor.reconcile(identity, 2, repository), /before state\/audit transaction commit/); + const crashed = await persistence.load(identity); + assert.equal(crashed?.activeTurn?.attemptId, 'attempt-live'); + assert.equal(crashed?.recoveryAttempt?.attemptId, 'recovery-before-crash'); + assert.equal((await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'attempt-live', + }, { type: 'output', channel: 'stdout', data: 'old output in crash window' })).accepted, true); + assert.deepEqual(await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'recovery-before-crash', + }, { type: 'output', channel: 'stdout', data: 'uncommitted replacement output' }), { + accepted: false, reason: 'turn_not_active', + }); + + const retried = await seeded.supervisor.reconcile(identity, 2, repository); + assert.equal(retried.outcome, 'resumed'); + assert.equal(retried.state.activeTurn?.attemptId, 'recovery-after-crash'); + assert.deepEqual(adapter.reconcileRequests.map(request => request.attemptId), [ + 'recovery-before-crash', 'recovery-after-crash', + ]); + assert.deepEqual(await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'attempt-live', + }, { type: 'output', channel: 'stdout', data: 'old output after promotion' }), { + accepted: false, reason: 'turn_not_active', + }); + assert.equal((await persistence.append({ ...fence, controllerEpoch: 2 }, { + executionId: 'execution-live', attemptId: 'recovery-after-crash', + }, { type: 'output', channel: 'stdout', data: 'retry replacement output' })).accepted, true); +}); + +test('recovered after-turn retry preserves a concurrent newer model intent and applies it on retry', async () => { + class BoundaryAdapter extends AddendumAdapter { + override readonly capabilities = { + nativeSessionId: 'eager', + steering: 'next_turn', + pause: 'after_turn', + modelChange: 'next_turn', + } as const; + readonly modelRequests: string[] = []; + modelStarted: (() => void) | undefined; + holdModel: Promise | undefined; + + override beginTurn(request: GoalBeginTurnRequest): AsyncIterable { + const adapter = this; + return (async function* () { + adapter.modelRequests.push(request.requestedModel); + adapter.modelStarted?.(); + if (adapter.holdModel) await adapter.holdModel; + if (request.modelChange) yield { + type: 'model_changed', model: request.requestedModel, + providerEventId: `accepted-${request.modelChange.modelChangeId}-${request.modelChange.generation}`, + } as const; + yield { type: 'completion', outcome: 'succeeded' } as const; + })(); + } + } + const adapter = new BoundaryAdapter(); + const persistence = new InMemoryGoalSessionPorts(); + const timestamp = new Date().toISOString(); + await persistence.create({ + ...identity, + provider: adapter.provider, + providerSessionId: 'owner-provider-session', + recoveryMetadata: { checkpoint: 'reconciled' }, + controllerEpoch: 1, + status: 'paused', + currentModel: 'model-a', + requestedModel: 'model-a', + recoveryAttemptId: 'attempt-reconciled', + activeTurn: { + executionId: 'execution-model-recovery', attemptId: 'attempt-reconciled', turnId: fence.turnId, + executionEpoch: 1, objective: 'continue with latest model', requestedModel: 'model-a', + repository, status: 'paused', + }, + completedTurnIds: [], + createdAt: timestamp, + updatedAt: timestamp, + }); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.requestModelChange({ ...identity, controllerEpoch: 1, model: 'model-old' }); + const modelStarted = deferred(); + const releaseOldModel = deferred(); + adapter.modelStarted = modelStarted.resolve; + adapter.holdModel = releaseOldModel.promise; + const staleResume = supervisor.resumeTurn({ ...identity, controllerEpoch: 1 }); + await modelStarted.promise; + + await supervisor.requestModelChange({ ...identity, controllerEpoch: 1, model: 'model-new' }); + releaseOldModel.resolve(); + const oldInvocation = await staleResume; + assert.equal(oldInvocation.disposition, 'started'); + const newerIntent = await persistence.load(identity); + assert.equal(newerIntent?.status, 'idle'); + assert.equal(newerIntent?.currentModel, 'model-old'); + assert.equal(newerIntent?.pendingModelChange, 'model-new'); + + adapter.modelStarted = undefined; + adapter.holdModel = undefined; + const recovered = await supervisor.runTurn({ + ...identity, controllerEpoch: 1, turnId: 'owner-next-turn', executionId: 'owner-next-execution', + attemptId: 'owner-next-attempt', objective: 'apply the newer deferred model', repository, + requestedModel: 'model-new', + }); + assert.equal(recovered.disposition, 'started'); + assert.equal(recovered.state.status, 'idle'); + assert.equal(recovered.state.currentModel, 'model-new'); + assert.equal(recovered.state.pendingModelChange, undefined); + assert.deepEqual(adapter.modelRequests, ['model-old', 'model-new']); + const replay = await persistence.replay(identity); + const acknowledgements = replay.filter(record => record.event.type === 'model_change_acknowledged'); + assert.deepEqual(acknowledgements.map(record => + record.event.type === 'model_change_acknowledged' ? record.event.requestedModel : ''), [ + 'model-old', 'model-new', + ]); + assert.deepEqual(replay.filter(record => record.event.type === 'model_changed').map(record => + record.event.type === 'model_changed' ? record.event.model : ''), ['model-old', 'model-new']); +}); diff --git a/packages/core/test/goalSessionProductionRecovery.test.ts b/packages/core/test/goalSessionProductionRecovery.test.ts new file mode 100644 index 000000000..9e5c6bce8 --- /dev/null +++ b/packages/core/test/goalSessionProductionRecovery.test.ts @@ -0,0 +1,388 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import Database from 'better-sqlite3'; +import knex from 'knex'; +import type { + GoalProviderOpenRequest, GoalProviderSessionSnapshot, GoalSessionAdapter, GoalSessionState, +} from '../src/agents/goalSession/contract.js'; +import { AuthoritativeGoalSessionRuntimePorts } from '../src/agents/goalSession/AuthoritativeGoalSessionRuntimePorts.js'; +import { GoalSessionContractError, GoalSessionScopeError } from '../src/agents/goalSession/errors.js'; +import { GoalSessionSupervisor } from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { GoalContainerSupervisor } from '../src/agents/goalSession/GoalContainerSupervisor.js'; +import { issueGoalSupervisedOpenPlan } from '../src/agents/goalSession/goalSessionOpen.js'; +import { startedProviderEffect } from '../src/agents/goalSession/providerEffectProtocol.js'; +import { rebuildMessageAcknowledgement, rebuildProviderSnapshot } from '../src/agents/goalSession/providerResultBoundary.js'; +import { + createSqliteGoalSessionRuntimePorts, SqliteGoalSessionControlDomain, +} from '../src/agents/goalSession/SqliteGoalSessionControlDomain.js'; +import { createProductionSchema, recovery, seedAuthoritativeGoal } from './productionGoalSessionTestSupport.js'; + +const identity = { goalId: 'production-goal', sessionId: 'production-session' }; + +function initialState(overrides: Partial> = {}): Omit { + const now = new Date().toISOString(); + return { + ...identity, provider: 'adapter', controllerEpoch: 1, status: 'initializing', + completedTurnIds: [], providerOpenAttemptId: 'open-attempt', + providerOpenOperationGeneration: 1, providerOperationGeneration: 1, + createdAt: now, updatedAt: now, ...overrides, + }; +} + +function runningState(): Omit { + return initialState({ + status: 'running', providerSessionId: 'provider-session', currentModel: 'model-a', + activeTurn: { + turnId: 'turn-one', executionId: 'execution-one', attemptId: 'attempt-one', executionEpoch: 1, + objective: 'exercise production fencing', requestedModel: 'model-a', + repository: { repository: 'integry/propr', worktreePath: '/tmp/worktree', branch: 'main' }, + status: 'running', providerOperationGeneration: 1, + }, + }); +} + +function openFence(operationId = 'open-attempt') { + return { ...identity, controllerEpoch: 1, generation: 1, kind: 'open' as const, operationId }; +} + +function steerFence() { + return { + ...identity, controllerEpoch: 1, generation: 1, kind: 'steer' as const, + operationId: 'message-one', turnId: 'turn-one', executionId: 'execution-one', attemptId: 'attempt-one', + }; +} + +test('actual #2018 and provider migrations compose in both orders, reopen, and rollback/up', async t => { + const foundation = await import('../src/db/migrations/20260831000000_create_goal_control_plane.js'); + const replay = await import('../src/db/migrations/20260901000000_add_durable_goal_replay.js'); + const runtime = await import('../src/db/migrations/20260902000000_extend_goal_control_provider_effects.js'); + for (const ordering of ['control-first', 'runtime-first'] as const) await t.test(ordering, async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-real-migrations-')); + const filename = path.join(directory, 'control.sqlite'); + const client = knex({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true }); + try { + if (ordering === 'runtime-first') await runtime.up(client); + await foundation.up(client); + await replay.up(client); + if (ordering === 'control-first') await runtime.up(client); + await runtime.down(client); + await runtime.up(client); + } finally { await client.destroy(); } + const first = new Database(filename); + seedAuthoritativeGoal(first, { goalId: identity.goalId, agent: 'adapter' }); + assert.doesNotThrow(() => new SqliteGoalSessionControlDomain(first)); + first.close(); + const reopened = new Database(filename); + assert.doesNotThrow(() => new SqliteGoalSessionControlDomain(reopened)); + assert.equal(reopened.prepare("SELECT 1 FROM sqlite_master WHERE name = 'goal_session_runtime_owners'").get(), undefined); + reopened.close(); + fs.rmSync(directory, { recursive: true, force: true }); + }); +}); + +test('goal_provider_sessions is the sole global owner and missing/wrong owners fail closed', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-global-owner-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const database = new Database(filename); + seedAuthoritativeGoal(database, { goalId: identity.goalId, agent: 'adapter' }); + seedAuthoritativeGoal(database, { goalId: 'other-goal', agent: 'adapter' }); + const domain = new SqliteGoalSessionControlDomain(database); + assert.ok(await domain.create(initialState())); + await assert.rejects(domain.load({ goalId: 'other-goal', sessionId: identity.sessionId }), GoalSessionScopeError); + await assert.rejects(domain.load({ goalId: identity.goalId, sessionId: 'missing-session' }), GoalSessionScopeError); + await assert.rejects(domain.replay({ goalId: 'other-goal', sessionId: identity.sessionId }), GoalSessionScopeError); + await assert.rejects(domain.claim({ goalId: 'other-goal', sessionId: identity.sessionId }, 'model-op', 'model-b'), GoalSessionScopeError); + await assert.rejects(domain.create(initialState({ goalId: 'other-goal' })), GoalSessionScopeError); + assert.deepEqual(database.prepare('SELECT session_id, goal_id FROM goal_provider_sessions').all(), [ + { session_id: identity.sessionId, goal_id: identity.goalId }, + ]); + database.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +test('two production supervisors recover and cancel one exact-label pending-open container', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-pending-open-owner-')); + const filename = path.join(directory, 'control.sqlite'); + const statePath = path.join(directory, 'container.state'); + const logPath = path.join(directory, 'docker.log'); + const dockerPath = path.join(import.meta.dirname, 'fixtures', 'fake-pending-open-docker.mjs'); + await createProductionSchema(filename); + const firstDatabase = new Database(filename); + seedAuthoritativeGoal(firstDatabase, { goalId: identity.goalId, agent: 'adapter' }); + const firstRuntime = createSqliteGoalSessionRuntimePorts(firstDatabase, recovery); + assert.ok(await firstRuntime.state.create(initialState())); + const secondDatabase = new Database(filename); + const secondRuntime = createSqliteGoalSessionRuntimePorts(secondDatabase, recovery); + const first = new GoalContainerSupervisor(directory, firstRuntime.events, undefined, { dockerPath }); + const second = new GoalContainerSupervisor(directory, secondRuntime.events, undefined, { dockerPath }); + const pendingIdentity = { + ...identity, attemptId: 'open-attempt', deterministicOpenKey: 'durable-open-key', + }; + fs.writeFileSync(statePath, 'pending-container'); + fs.writeFileSync(logPath, ''); + const previous = { + state: process.env.GOAL_PENDING_OPEN_STATE, + log: process.env.GOAL_PENDING_OPEN_LOG, + labels: process.env.GOAL_PENDING_OPEN_LABELS, + }; + process.env.GOAL_PENDING_OPEN_STATE = statePath; + process.env.GOAL_PENDING_OPEN_LOG = logPath; + process.env.GOAL_PENDING_OPEN_LABELS = JSON.stringify({ + 'propr.goal.id': identity.goalId, + 'propr.goal.session': identity.sessionId, + 'propr.goal.scope': 'open', + 'propr.goal.attempt': pendingIdentity.attemptId, + 'propr.goal.open-key': pendingIdentity.deterministicOpenKey, + }); + try { + await Promise.all([ + first.cancelPendingOpenAttempt(pendingIdentity), + second.cancelPendingOpenAttempt(pendingIdentity), + ]); + } finally { + restoreEnvironment('GOAL_PENDING_OPEN_STATE', previous.state); + restoreEnvironment('GOAL_PENDING_OPEN_LOG', previous.log); + restoreEnvironment('GOAL_PENDING_OPEN_LABELS', previous.labels); + } + assert.equal(fs.existsSync(statePath), false); + const calls = fs.readFileSync(logPath, 'utf8').trim().split('\n').map(line => JSON.parse(line) as string[]); + assert.ok(calls.some(call => call[0] === 'rm' && call[2] === 'pending-container')); + assert.ok(calls.filter(call => call[0] === 'ps').every(call => call.includes('label=propr.goal.scope=open'))); + firstDatabase.close(); secondDatabase.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +test('token-fenced production effects admit one non-open callback and replay only settled DTOs', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-effect-token-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const firstDatabase = new Database(filename); + seedAuthoritativeGoal(firstDatabase, { goalId: identity.goalId, agent: 'adapter' }); + const firstDomain = new SqliteGoalSessionControlDomain(firstDatabase); + assert.ok(await firstDomain.create(runningState())); + const secondDatabase = new Database(filename); + const first = new AuthoritativeGoalSessionRuntimePorts(firstDomain, recovery); + const duplicate = new AuthoritativeGoalSessionRuntimePorts(new SqliteGoalSessionControlDomain(secondDatabase), recovery); + let resolve!: (value: { messageId: string }) => void; + const completion = new Promise<{ messageId: string }>(done => { resolve = done; }); + let callbacks = 0; + const delivery = first.start(steerFence(), 'provider_primitive', () => { + callbacks += 1; + return startedProviderEffect(completion, () => undefined); + }, rebuildMessageAcknowledgement); + await new Promise(done => setImmediate(done)); + await assert.rejects(duplicate.start(steerFence(), 'provider_primitive', () => { + callbacks += 1; + return startedProviderEffect(Promise.resolve({ messageId: 'wrong' }), () => undefined); + }, rebuildMessageAcknowledgement), providerDoubt); + resolve({ messageId: 'message-one' }); + assert.deepEqual(await delivery, { messageId: 'message-one' }); + let replayedCallback = false; + assert.deepEqual(await duplicate.start(steerFence(), 'provider_primitive', () => { + replayedCallback = true; + return startedProviderEffect(Promise.resolve({ messageId: 'wrong' }), () => undefined); + }, rebuildMessageAcknowledgement), { messageId: 'message-one' }); + assert.deepEqual({ callbacks, replayedCallback }, { callbacks: 1, replayedCallback: false }); + firstDatabase.close(); secondDatabase.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +test('every internal effect stage is independently fenced across two SQLite connections', async t => { + for (const stage of ['provider_primitive', 'stream_first_next', 'container_spawn'] as const) await t.test(stage, async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-effect-stage-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const database = new Database(filename); + seedAuthoritativeGoal(database, { goalId: identity.goalId, agent: 'adapter' }); + const domain = new SqliteGoalSessionControlDomain(database); + assert.ok(await domain.create(initialState())); + const peerDatabase = new Database(filename); + const gate = new AuthoritativeGoalSessionRuntimePorts(domain, recovery); + const peer = new AuthoritativeGoalSessionRuntimePorts(new SqliteGoalSessionControlDomain(peerDatabase), recovery); + let callbacks = 0; + const first = await gate.start(openFence(), stage, () => { + callbacks += 1; + return startedProviderEffect(Promise.resolve({ messageId: stage }), () => undefined); + }, rebuildMessageAcknowledgement); + assert.equal(first.messageId, stage); + if (stage === 'container_spawn') { + await assert.rejects(peer.start(openFence(), stage, () => { + callbacks += 1; + return startedProviderEffect(Promise.resolve({ messageId: 'duplicate' }), () => undefined); + }, rebuildMessageAcknowledgement), providerDoubt); + } else { + assert.deepEqual(await peer.start(openFence(), stage, () => { + callbacks += 1; + return startedProviderEffect(Promise.resolve({ messageId: 'duplicate' }), () => undefined); + }, rebuildMessageAcknowledgement), first); + } + assert.equal(callbacks, 1); + database.close(); peerDatabase.close(); + fs.rmSync(directory, { recursive: true, force: true }); + }); +}); + +test('hostile and lossy provider values poison the exact token before settlement', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-hostile-result-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const database = new Database(filename); + seedAuthoritativeGoal(database, { goalId: identity.goalId, agent: 'adapter' }); + const domain = new SqliteGoalSessionControlDomain(database); + assert.ok(await domain.create(initialState())); + const gate = new AuthoritativeGoalSessionRuntimePorts(domain, recovery); + let toJsonCalls = 0; + const hostile = { + providerSessionId: 'provider-session', recoveryMetadata: {}, + toJSON() { toJsonCalls += 1; return { providerSessionId: 'clean', recoveryMetadata: {} }; }, + }; + let callbacks = 0; + await assert.rejects(gate.start(openFence(), 'provider_primitive', () => { + callbacks += 1; + return startedProviderEffect(Promise.resolve(hostile), () => undefined); + }, value => rebuildProviderSnapshot(value, 'adapter')), /invalid session snapshot/); + await assert.rejects(gate.start(openFence(), 'provider_primitive', () => { + callbacks += 1; + return startedProviderEffect(Promise.resolve({ providerSessionId: 'other', recoveryMetadata: {} }), () => undefined); + }, value => rebuildProviderSnapshot(value, 'adapter')), providerDoubt); + assert.deepEqual({ callbacks, toJsonCalls }, { callbacks: 1, toJsonCalls: 0 }); + assert.equal((database.prepare('SELECT status FROM goal_session_runtime_provider_effects').get() as { status: string }).status, 'poisoned'); + database.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +test('closed fences, 255-byte IDs, allocator replay, and message state obey #2018', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-boundaries-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const database = new Database(filename); + seedAuthoritativeGoal(database, { goalId: identity.goalId, agent: 'adapter' }); + const domain = new SqliteGoalSessionControlDomain(database); + assert.ok(await domain.create(runningState())); + const beforeEffects = database.prepare('SELECT COUNT(*) AS count FROM goal_session_runtime_provider_effects').get() as { count: number }; + await assert.rejects(domain.claimProviderEffect({ ...steerFence(), kind: 'future' } as never, 'provider_primitive')); + await assert.rejects(domain.claimProviderEffect({ ...steerFence(), excess: true } as never, 'provider_primitive')); + await assert.rejects(domain.claimProviderEffect(steerFence(), 'future' as never)); + assert.equal((database.prepare('SELECT COUNT(*) AS count FROM goal_session_runtime_provider_effects').get() as { count: number }).count, beforeEffects.count); + + const state = await domain.load(identity); + assert.ok(state?.activeTurn); + const appended = await domain.append( + { ...identity, controllerEpoch: 1, turnId: 'turn-one' }, + { executionId: 'execution-one', attemptId: 'attempt-one' }, + { type: 'output', channel: 'stdout', data: 'hello' }, + ); + assert.equal(appended.accepted, true); + const eventRow = database.prepare('SELECT sequence, kind, event_type FROM goal_events').get() as Record; + assert.deepEqual(eventRow, { sequence: 1, kind: 'domain', event_type: 'goal_session.output' }); + assert.equal((database.prepare('SELECT high_watermark FROM goal_event_state WHERE goal_id = ?').get(identity.goalId) as { high_watermark: number }).high_watermark, 1); + database.prepare(`INSERT INTO goal_events + (goal_id, sequence, kind, event_type, payload_json, idempotency_key, lease_epoch, created_at, + schema_version, payload_bytes) + VALUES (?, 2, 'domain', 'unrelated.event', NULL, 'unrelated-null', 1, ?, 1, 0)`) + .run(identity.goalId, new Date().toISOString()); + database.prepare('UPDATE goal_event_state SET high_watermark = 2 WHERE goal_id = ?').run(identity.goalId); + assert.equal((await domain.replay(identity)).length, 1); + + database.prepare(`INSERT INTO goal_messages + (message_id, goal_id, sequence, queue_ordinal, body, state, delivery_attempts, retry_count, + idempotency_key, created_at) + VALUES ('message-one', ?, 1, 1, 'corrective', 'queued', 0, 0, 'message-key', ?)`) + .run(identity.goalId, new Date().toISOString()); + assert.equal(await domain.acknowledgeWithEvent( + { ...identity, controllerEpoch: 1, turnId: 'turn-one' }, + { executionId: 'execution-one', attemptId: 'attempt-one' }, 'message-one', + ), 'acknowledged'); + const message = database.prepare(`SELECT state, delivered_at, acknowledged_at + FROM goal_messages WHERE message_id = 'message-one'`).get() as Record; + assert.equal(message.state, 'acknowledged'); + assert.equal(typeof message.delivered_at, 'string'); + assert.equal(typeof message.acknowledged_at, 'string'); + + const longGoal = `g${'a'.repeat(254)}`; + const longSession = `s${'b'.repeat(254)}`; + seedAuthoritativeGoal(database, { goalId: longGoal, agent: 'adapter' }); + assert.ok(await domain.create(initialState({ goalId: longGoal, sessionId: longSession }))); + await assert.rejects(domain.create(initialState({ + goalId: `g${'a'.repeat(255)}`, sessionId: `s${'b'.repeat(255)}`, + })), GoalSessionScopeError); + database.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +test('settled successful open survives state-CAS crash with one total thread/start', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-open-settled-')); + const filename = path.join(directory, 'control.sqlite'); + await createProductionSchema(filename); + const database = new Database(filename); + seedAuthoritativeGoal(database, { goalId: identity.goalId, agent: 'codex', model: 'gpt-5.6-sol' }); + const runtime = createSqliteGoalSessionRuntimePorts(database, recovery); + let starts = 0; + const adapter = codexAdapter(() => { starts += 1; }); + const plan = issueGoalSupervisedOpenPlan({ + repository: { repository: 'integry/propr', worktreePath: '/tmp/worktree', branch: 'main' }, + requestedModel: 'gpt-5.6-sol', providerHomeTarget: '/home/node/.codex', credentialTargets: [], + }, { + createTransport: async () => inertTransport(), cancelPending: async () => undefined, + transferPending: () => undefined, + }); + const originalCompareAndSet = runtime.state.compareAndSet.bind(runtime.state); + let crash = true; + runtime.state.compareAndSet = async (expected, next) => { + if (crash && next.providerSessionId) { crash = false; return null; } + return originalCompareAndSet(expected, next); + }; + const first = new GoalSessionSupervisor(adapter, runtime, () => 'open-attempt'); + await assert.rejects(first.openSession({ ...identity, provider: 'codex', controllerEpoch: 1, supervisedOpen: plan })); + const replacement = new GoalSessionSupervisor(adapter, runtime, () => 'replacement-attempt'); + const reopened = await replacement.openSession({ ...identity, provider: 'codex', controllerEpoch: 1, supervisedOpen: plan }); + assert.equal(reopened.providerSessionId, 'codex-thread'); + assert.equal(starts, 1, 'thread/start is replayed from the exact settled operation'); + database.close(); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); +}); + +function providerDoubt(error: unknown): boolean { + return error instanceof GoalSessionContractError && error.code === 'PROVIDER_EFFECT_IN_DOUBT'; +} + +function restoreEnvironment(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +function codexAdapter(onStart: () => void): GoalSessionAdapter { + return { + provider: 'codex', supportsDeterministicOpen: true, + capabilities: { nativeSessionId: 'eager', steering: 'active_turn', pause: 'after_turn', modelChange: 'next_turn' }, + publishOperationBarrier: async () => undefined, + openSession: async (request: GoalProviderOpenRequest): Promise => { + onStart(); + return { + providerSessionId: 'codex-thread', model: 'gpt-5.6-sol', recoveryMetadata: { + version: 2, provider: 'codex', protocolVersion: 'app-server-0.146.0', payload: { + threadId: 'codex-thread', sessionId: 'codex-session', initialized: true, + openKey: request.deterministicOpenKey!, repository: 'integry/propr', model: 'gpt-5.6-sol', + providerHomeIdentity: '/home/node/.codex', cliVersion: '0.146.0', + }, usage: { components: [] }, + }, + }; + }, + beginTurn: async function* () {}, resumeSession: async (_request, snapshot) => snapshot, + requestModelChange: async request => ({ requestedModel: request.model, appliesAt: 'next_turn' }), + cancel: async () => undefined, cancelPending: async () => undefined, + reconcile: async () => ({ outcome: 'failed', reason: 'unused' }), + }; +} + +function inertTransport() { + return { + output: { async *[Symbol.asyncIterator]() {} }, write: async () => undefined, + closeInput: () => undefined, cancel: async () => undefined, + completion: Promise.resolve({ exitCode: 0 }), + }; +} diff --git a/packages/core/test/goalSessionQueuedOwnerAddendum.test.ts b/packages/core/test/goalSessionQueuedOwnerAddendum.test.ts new file mode 100644 index 000000000..9bb897781 --- /dev/null +++ b/packages/core/test/goalSessionQueuedOwnerAddendum.test.ts @@ -0,0 +1,285 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalModelChangeIntent, + GoalProviderModelChangeRequest, + GoalProviderReconcileRequest, + GoalSessionAdapter, + GoalSessionEvent, +} from '../src/agents/goalSession/contract.js'; +import { GoalSessionSupervisor } from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { + compactImmediateModelIntents, + MODEL_CHANGE_SETTLED_RETRY_HORIZON, +} from '../src/agents/goalSession/modelChangeProtocol.js'; +import { + fingerprintGoalWorktree, + normalizeGitRepositoryIdentity, +} from '../src/agents/goalSession/worktreeIdentity.js'; +import { SqliteGoalSessionTestPorts } from './SqliteGoalSessionTestPorts.js'; + +const identity = { goalId: 'queued-addendum-goal', sessionId: 'queued-addendum-session' }; +const repository = { + repository: 'integry/propr', worktreePath: '/tmp/queued-addendum-worktree', branch: 'follow-up', +}; + +class AddendumAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'queued-addendum-provider'; + readonly capabilities = { + nativeSessionId: 'eager' as const, + steering: 'active_turn' as const, + pause: 'active_turn' as const, + modelChange: 'next_safe_boundary' as const, + }; + readonly modelCalls: GoalProviderModelChangeRequest[] = []; + readonly reconcileRequests: GoalProviderReconcileRequest[] = []; + readonly beginRequests: GoalBeginTurnRequest[] = []; + currentModel = 'model-base'; + + async openSession() { + return { providerSessionId: 'queued-native', recoveryMetadata: { checkpoint: 'open' }, model: this.currentModel }; + } + + beginTurn(request: GoalBeginTurnRequest): AsyncIterable { + this.beginRequests.push(structuredClone(request)); + return (async function* () { yield { type: 'completion', outcome: 'succeeded' } as const; })(); + } + + async resumeSession(_request: unknown, snapshot: { providerSessionId: string; recoveryMetadata: object }) { + return snapshot; + } + + async requestModelChange(request: GoalProviderModelChangeRequest) { + this.modelCalls.push(structuredClone(request)); + this.currentModel = request.model; + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + + async cancel() {} + + async reconcile(request: GoalProviderReconcileRequest) { + this.reconcileRequests.push(structuredClone(request)); + return { outcome: 'alive' as const, reason: 'authoritative recovery target matched' }; + } +} + +function temporaryDatabase(): { filename: string; cleanup: () => void } { + const directory = mkdtempSync(join(tmpdir(), 'goal-model-retention-')); + return { filename: join(directory, 'state.sqlite'), cleanup: () => rmSync(directory, { recursive: true, force: true }) }; +} + +test('compaction retains every unresolved generation and the newest settled retry horizon in order', () => { + const unresolved = new Set([7, 89, 151]); + const intents: GoalModelChangeIntent[] = Array.from({ length: 240 }, (_, offset) => { + const generation = offset + 1; + return { + modelChangeId: `change-${generation}`, + model: `model-${generation}`, + requestedAt: new Date(generation * 1000).toISOString(), + generation, + phase: unresolved.has(generation) ? 'provider_in_doubt' : 'superseded', + acknowledgement: { requestedModel: `model-${generation}`, appliesAt: 'immediate' }, + }; + }); + const compacted = compactImmediateModelIntents(intents); + + assert.deepEqual(compacted.filter(intent => !['committed', 'superseded'].includes(intent.phase ?? '')) + .map(intent => intent.generation), [...unresolved]); + assert.deepEqual(compacted.filter(intent => intent.phase === 'superseded').map(intent => intent.generation), + Array.from({ length: MODEL_CHANGE_SETTLED_RETRY_HORIZON }, (_, offset) => 177 + offset)); + assert.deepEqual(compacted.map(intent => intent.generation), + [...compacted.map(intent => intent.generation)].sort((left, right) => left! - right!)); +}); + +test('reopen compacts an oversized settled ledger from an older process without losing the latest model', async () => { + const adapter = new AddendumAdapter(); + adapter.currentModel = 'model-300'; + const ports = new InMemoryGoalSessionPorts(); + const timestamp = new Date().toISOString(); + const intents: GoalModelChangeIntent[] = Array.from({ length: 300 }, (_, offset) => ({ + modelChangeId: `legacy-${offset + 1}`, + model: `model-${offset + 1}`, + requestedAt: timestamp, + generation: offset + 1, + phase: 'committed', + acknowledgement: { + requestedModel: `model-${offset + 1}`, appliesAt: 'immediate', effectiveModel: `model-${offset + 1}`, + }, + })); + await ports.create({ + ...identity, provider: adapter.provider, providerSessionId: 'queued-native', recoveryMetadata: {}, + controllerEpoch: 1, status: 'idle', currentModel: 'model-300', requestedModel: 'model-300', + completedTurnIds: [], modelChangeGeneration: 300, modelChangeIntents: intents, + modelChangeIntent: intents.at(-1), createdAt: timestamp, updatedAt: timestamp, + }); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + const reopened = await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }); + + assert.equal(reopened.currentModel, 'model-300'); + assert.equal(reopened.requestedModel, 'model-300'); + assert.equal(reopened.modelChangeGeneration, 300); + assert.equal(reopened.modelChangeIntents?.length, MODEL_CHANGE_SETTLED_RETRY_HORIZON); + assert.equal(reopened.modelChangeIntent?.modelChangeId, 'legacy-300'); +}); + +test('thousands of model switches stay bounded across a crash, takeover, cached retry, and SQLite reopen', async t => { + const database = temporaryDatabase(); + t.after(database.cleanup); + const adapter = new AddendumAdapter(); + let ports = new SqliteGoalSessionTestPorts(database.filename); + let supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + + let sizeAtHorizon = 0; + for (let generation = 0; generation < 5_001; generation += 1) { + const model = `model-${generation.toString().padStart(4, '0')}`; + if (generation === 80) { + ports.setTransitionFault('before_commit'); + await assert.rejects(supervisor.requestModelChange({ ...identity, controllerEpoch: 1, model }), + /Injected crash/); + const unresolved = await ports.load(identity); + assert.equal(unresolved?.modelChangeIntents?.at(-1)?.phase, 'provider_in_doubt'); + assert.ok((unresolved?.modelChangeIntents?.length ?? 0) <= MODEL_CHANGE_SETTLED_RETRY_HORIZON + 1); + ports.close(); + ports = new SqliteGoalSessionTestPorts(database.filename); + supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }); + } else { + await supervisor.requestModelChange({ + ...identity, controllerEpoch: generation < 80 ? 1 : 2, model, + }); + } + if (generation === 100) sizeAtHorizon = JSON.stringify(await ports.load(identity)).length; + } + + const settled = await ports.load(identity); + assert.equal(settled?.currentModel, 'model-5000'); + assert.equal(settled?.requestedModel, 'model-5000'); + assert.equal(settled?.modelChangeGeneration, 5_001); + assert.equal(settled?.modelChangeIntents?.length, MODEL_CHANGE_SETTLED_RETRY_HORIZON); + assert.equal(settled?.modelChangeIntents?.at(0)?.generation, 4_938); + assert.equal(settled?.modelChangeIntents?.at(-1)?.generation, 5_001); + assert.ok(JSON.stringify(settled).length <= sizeAtHorizon + 2_048); + assert.ok(JSON.stringify(settled).length < 32_000); + + const latestId = settled?.modelChangeIntents?.at(-1)?.modelChangeId; + await supervisor.requestModelChange({ ...identity, controllerEpoch: 2, model: 'model-5000' }); + assert.equal(adapter.modelCalls.at(-1)?.modelChangeId, latestId, 'cached retry keeps its provider idempotency identity'); + assert.equal(adapter.modelCalls.at(-1)?.applicationGeneration, 5_001); + + const events = await ports.replay(identity); + assert.equal(events.filter(record => record.event.type === 'model_change_acknowledged').length, 5_001); + assert.equal(events.filter(record => record.event.type === 'model_changed').length, 5_001); + assert.equal(events.find(record => record.event.type === 'model_changed')?.event.type === 'model_changed' + ? events.find(record => record.event.type === 'model_changed')!.event.model : undefined, 'model-0000'); + + const firstOperationId = adapter.modelCalls[0]?.modelChangeId; + assert.ok(firstOperationId); + assert.equal((await supervisor.requestModelChange({ + ...identity, controllerEpoch: 2, model: 'model-0000', operationId: firstOperationId, + })).outcome, 'outside_retry_horizon'); + const retained = settled?.modelChangeIntents?.at(0); + assert.ok(retained); + assert.equal((await supervisor.requestModelChange({ + ...identity, controllerEpoch: 2, model: retained.model, operationId: retained.modelChangeId, + })).requestedModel, retained.model); + const neverIssued = await supervisor.requestModelChange({ + ...identity, controllerEpoch: 2, model: 'model-never-issued', operationId: 'adversarial-never-issued-after-5001', + }); + assert.notEqual(neverIssued.outcome, 'outside_retry_horizon'); + await assert.rejects(supervisor.requestModelChange({ + ...identity, controllerEpoch: 2, model: 'model-conflict', operationId: 'adversarial-never-issued-after-5001', + }), /different model/); + ports.close(); +}); + +test('Git remotes normalize canonical identities and reject credential-bearing or malformed values', () => { + const accepted = new Map([ + ['git@github.com:integry/propr.git', 'integry/propr'], + ['https://github.com/integry/propr.git', 'integry/propr'], + ]); + for (const [remote, expected] of accepted) assert.equal(normalizeGitRepositoryIdentity(remote), expected); + for (const remote of [ + 'https://alice:secret@/integry/propr', + 'https://alice:token-value@github.com/integry/propr.git', + 'ssh://git:private-key@github.com/integry/propr.git', + 'token-user@github.com:integry/propr.git?access_token=query-secret', + 'https://oauth2:secret@gitlab.example.com/group/project.git', + 'file:///tmp/credential/repository', + 'alice:secret@github.com/integry/propr', + 'https://github.com', + 'https://github.com/integry/repository\nsecret', + ]) assert.equal(normalizeGitRepositoryIdentity(remote), undefined); +}); + +test('turn and recovery boundaries never expose credential-bearing remotes in provider, state, event, or error data', async () => { + const token = 'TOP-SECRET-GIT-TOKEN'; + const credentialRemote = `https://owner:${token}@github.com/integry/propr.git`; + const adapter = new AddendumAdapter(); + const turnPorts = new InMemoryGoalSessionPorts(); + const turnSupervisor = new GoalSessionSupervisor(adapter, turnPorts.asRuntimePorts()); + await turnSupervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + await assert.rejects(turnSupervisor.runTurn({ + ...identity, controllerEpoch: 1, turnId: 'credential-turn', executionId: 'credential-execution', + attemptId: 'credential-attempt', objective: 'scrub remote', + repository: { + ...repository, + repository: credentialRemote, + credentialBearingRemote: credentialRemote, + } as typeof repository, + requestedModel: 'model-base', + }), /trustworthy Git repository/); + assert.equal(adapter.beginRequests.length, 0); + + const recoveryIdentity = { goalId: 'credential-recovery-goal', sessionId: 'credential-recovery-session' }; + const recoveryPorts = new InMemoryGoalSessionPorts(); + const timestamp = new Date().toISOString(); + await recoveryPorts.create({ + ...recoveryIdentity, provider: adapter.provider, providerSessionId: 'queued-native', recoveryMetadata: {}, + controllerEpoch: 1, status: 'running', currentModel: 'model-base', completedTurnIds: [], + activeTurn: { + turnId: 'recovery-turn', executionId: 'recovery-execution', attemptId: 'recovery-attempt', executionEpoch: 1, + objective: 'recover securely', requestedModel: 'model-base', repository, status: 'running', + }, + createdAt: timestamp, updatedAt: timestamp, + }); + recoveryPorts.setRepositoryInspection(repository, { + ...repository, exists: true, observedRepository: credentialRemote, observedBranch: repository.branch, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + reason: `untrusted diagnostic ${credentialRemote}`, + }); + const recoverySupervisor = new GoalSessionSupervisor(adapter, recoveryPorts.asRuntimePorts()); + const recoveryResult = await recoverySupervisor.reconcile(recoveryIdentity, 1, repository); + assert.equal(recoveryResult.outcome, 'blocked'); + assert.equal(adapter.reconcileRequests.length, 0); + + let invalidError = ''; + try { + await turnSupervisor.runTurn({ + ...identity, controllerEpoch: 1, turnId: 'invalid-turn', executionId: 'invalid-execution', + objective: 'reject malformed remote', repository: { + ...repository, repository: `https://owner:${token}@/integry/propr`, + }, requestedModel: 'model-base', + }); + } catch (error) { + invalidError = error instanceof Error ? error.message : String(error); + } + const boundaryData = JSON.stringify({ + beginRequests: adapter.beginRequests, + reconcileRequests: adapter.reconcileRequests, + turnState: await turnPorts.load(identity), + turnEvents: await turnPorts.replay(identity), + recoveryState: await recoveryPorts.load(recoveryIdentity), + recoveryEvents: await recoveryPorts.replay(recoveryIdentity), + invalidError, + }); + assert.doesNotMatch(boundaryData, new RegExp(token)); + assert.doesNotMatch(boundaryData, /owner:TOP-SECRET/); + assert.match(invalidError, /trustworthy Git repository/); +}); diff --git a/packages/core/test/goalSessionReaudit.test.ts b/packages/core/test/goalSessionReaudit.test.ts new file mode 100644 index 000000000..6ba998b0c --- /dev/null +++ b/packages/core/test/goalSessionReaudit.test.ts @@ -0,0 +1,429 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalProviderCancelRequest, + GoalProviderCapabilities, + GoalProviderModelChangeRequest, + GoalProviderOpenRequest, + GoalProviderReconcileRequest, + GoalProviderSessionSnapshot, + GoalSessionAdapter, + GoalSessionControlFence, + GoalSessionControlTransition, + GoalSessionEvent, + GoalSessionState, + GoalTerminalCommit, +} from '../src/agents/goalSession/contract.js'; +import { + GoalSessionSupervisor, + StaleGoalSessionFenceError, +} from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; + +const identity = { goalId: 'reaudit-goal', sessionId: 'reaudit-session' }; +const repository = { + repository: 'integry/propr', worktreePath: '/tmp/reaudit-worktree', branch: 'reaudit-branch', headSha: 'reaudit-head', +}; +const control = { ...identity, controllerEpoch: 1 }; +const turnFence = { ...control, turnId: 'reaudit-turn' }; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +class ReauditAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'reaudit-provider'; + readonly capabilities: GoalProviderCapabilities; + readonly modelCalls: GoalProviderModelChangeRequest[] = []; + readonly turnCalls: GoalBeginTurnRequest[] = []; + readonly turnModelEffects = new Set(); + readonly cancelCalls: GoalProviderCancelRequest[] = []; + readonly modelEffects = new Set(); + readonly cancelEffects: Set; + openCalls = 0; + stream: (request: GoalBeginTurnRequest) => AsyncIterable = async function* () { + yield { type: 'completion', outcome: 'succeeded' }; + }; + cancelGate: Promise | undefined; + cancelStarted: (() => void) | undefined; + + constructor( + capabilities: GoalProviderCapabilities = { + nativeSessionId: 'eager', steering: 'active_turn', pause: 'active_turn', modelChange: 'next_safe_boundary', + }, + cancelEffects = new Set(), + ) { + this.capabilities = capabilities; + this.cancelEffects = cancelEffects; + } + + async openSession(_request: GoalProviderOpenRequest): Promise { + this.openCalls += 1; + return { providerSessionId: 'reaudit-native', recoveryMetadata: { checkpoint: 'open' }, model: 'model-a' }; + } + + beginTurn(request: GoalBeginTurnRequest): AsyncIterable { + this.turnCalls.push(structuredClone(request)); + if (request.modelChange) this.turnModelEffects.add(request.modelChange.modelChangeId); + const stream = this.stream(request); + return (async function* () { + if (request.modelChange) yield { + type: 'model_changed' as const, + model: request.requestedModel, + providerEventId: `accepted-${request.modelChange.modelChangeId}-${request.modelChange.generation}`, + }; + yield* stream; + })(); + } + + async resumeSession( + _request: GoalSessionControlFence, + snapshot: GoalProviderSessionSnapshot, + ): Promise { return snapshot; } + + async requestModelChange(request: GoalProviderModelChangeRequest) { + this.modelCalls.push(structuredClone(request)); + this.modelEffects.add(request.modelChangeId); + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + + async cancel(request: GoalProviderCancelRequest): Promise { await this.signalCancel(request); } + + async cancelPending(request: GoalProviderCancelRequest): Promise { await this.signalCancel(request); } + + async reconcile(_request: GoalProviderReconcileRequest) { + return { outcome: 'alive' as const, reason: 'alive' }; + } + + private async signalCancel(request: GoalProviderCancelRequest): Promise { + this.cancelCalls.push(structuredClone(request)); + this.cancelEffects.add(request.cancellationId); + this.cancelStarted?.(); + if (this.cancelGate) await this.cancelGate; + } +} + +async function openRuntime(adapter: ReauditAdapter, persistence = new InMemoryGoalSessionPorts()) { + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + return { persistence, supervisor }; +} + +function turnRequest(model = 'model-a') { + return { + ...turnFence, + executionId: 'reaudit-execution', + attemptId: 'reaudit-attempt', + objective: 'exercise the audited race', + repository, + requestedModel: model, + }; +} + +test('cancellation claim immediately fences FIFO acknowledgement and output through provider races and repeat cancel', async () => { + const adapter = new ReauditAdapter({ + nativeSessionId: 'eager', steering: 'next_turn', pause: 'active_turn', modelChange: 'next_safe_boundary', + }); + const streamStarted = deferred(); + const releaseStream = deferred(); + const cancelStarted = deferred(); + const releaseCancel = deferred(); + adapter.stream = async function* (request) { + streamStarted.resolve(); + await releaseStream.promise; + yield { type: 'message_acknowledged', messageId: request.correctiveMessages![0].messageId }; + yield { type: 'output', channel: 'stdout', data: 'must be fenced' }; + yield { type: 'completion', outcome: 'succeeded' }; + }; + adapter.cancelStarted = cancelStarted.resolve; + adapter.cancelGate = releaseCancel.promise; + const { persistence, supervisor } = await openRuntime(adapter); + persistence.enqueueMessage({ ...identity, messageId: 'reaudit-message', body: 'correct this' }); + const running = supervisor.runTurn(turnRequest()); + await streamStarted.promise; + + const firstCancel = supervisor.cancel({ ...control, reason: 'cancel now' }); + await cancelStarted.promise; + const cancelling = await persistence.load(identity); + assert.equal(cancelling?.status, 'cancelling'); + assert.equal(cancelling?.activeTurn, undefined, 'the durable claim clears live attempt ownership before provider await'); + const oldExecution = { executionId: 'reaudit-execution', attemptId: 'reaudit-attempt' }; + assert.equal(await persistence.acknowledge(turnFence, oldExecution, 'reaudit-message'), 'stale_fence'); + assert.deepEqual(await persistence.append(turnFence, oldExecution, { + type: 'output', channel: 'stdout', data: 'direct stale output', + }), { accepted: false, reason: 'turn_not_active' }); + + const repeatedCancel = supervisor.cancel({ ...control, reason: 'different retry text must not replace the claim' }); + for (let attempt = 0; attempt < 20 && adapter.cancelCalls.length < 2; attempt += 1) await Promise.resolve(); + assert.equal(adapter.cancelCalls.length, 2); + assert.equal(new Set(adapter.cancelCalls.map(call => call.cancellationId)).size, 1); + assert.equal(adapter.cancelEffects.size, 1, 'the stable provider key deduplicates the cancellation primitive'); + + releaseStream.resolve(); + await assert.rejects(running, StaleGoalSessionFenceError); + assert.deepEqual((await persistence.listPending(identity)).map(message => message.messageId), ['reaudit-message']); + assert.equal((await persistence.replay(identity)).some(record => + record.event.type === 'message_acknowledged' || record.event.type === 'output'), false); + + releaseCancel.resolve(); + const [firstTerminal, repeatedTerminal] = await Promise.all([firstCancel, repeatedCancel]); + assert.equal(firstTerminal.status, 'terminated'); + assert.equal(repeatedTerminal.status, 'terminated'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('process replacement open resumes bound and unbound cancelling claims without opening work', async t => { + for (const binding of ['bound', 'unbound'] as const) { + await t.test(binding, async () => { + const capabilities: GoalProviderCapabilities = binding === 'bound' + ? { nativeSessionId: 'eager', steering: 'active_turn', pause: 'active_turn', modelChange: 'next_safe_boundary' } + : { + nativeSessionId: 'first_turn', firstTurnIdCrashPolicy: 'retry_deterministically', + steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + }; + const effects = new Set(); + const initial = new ReauditAdapter(capabilities, effects); + const initialStarted = deferred(); + const releaseInitial = deferred(); + initial.cancelStarted = initialStarted.resolve; + initial.cancelGate = releaseInitial.promise; + const persistence = new InMemoryGoalSessionPorts(); + const firstSupervisor = new GoalSessionSupervisor(initial, persistence.asRuntimePorts()); + await firstSupervisor.openSession({ ...identity, provider: initial.provider, controllerEpoch: 1 }); + const originalCancel = firstSupervisor.cancel({ ...control, reason: `${binding} crash cancellation` }); + await initialStarted.promise; + + const replacement = new ReauditAdapter(capabilities, effects); + const replacementStarted = deferred(); + const releaseReplacement = deferred(); + replacement.cancelStarted = replacementStarted.resolve; + replacement.cancelGate = releaseReplacement.promise; + const reopenedSupervisor = new GoalSessionSupervisor(replacement, persistence.asRuntimePorts()); + const reopening = reopenedSupervisor.openSession({ ...identity, provider: replacement.provider, controllerEpoch: 2 }); + await replacementStarted.promise; + assert.equal(replacement.openCalls, 0, 'reopen must resume cancellation rather than open/resume provider work'); + assert.equal((await persistence.load(identity))?.activeTurn, undefined); + assert.equal(effects.size, 1); + assert.equal(initial.cancelCalls[0].cancellationId, replacement.cancelCalls[0].cancellationId); + + releaseInitial.resolve(); + assert.equal((await originalCancel).status, 'terminated'); + releaseReplacement.resolve(); + assert.equal((await reopening).status, 'terminated'); + assert.equal((await reopenedSupervisor.openSession({ + ...identity, provider: replacement.provider, controllerEpoch: 2, + })).status, 'terminated'); + assert.equal(replacement.openCalls, 0); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); + }); + } +}); + +class TakeoverCompletionRacePorts extends InMemoryGoalSessionPorts { + beforeTakeover: (() => Promise) | undefined; + + override async compareAndSet(expected: GoalSessionState, next: Omit) { + if (next.controllerEpoch > expected.controllerEpoch && this.beforeTakeover) { + const hook = this.beforeTakeover; + this.beforeTakeover = undefined; + await hook(); + } + return super.compareAndSet(expected, next); + } +} + +test('reopen converges when provider cancellation completes in the takeover CAS window', async () => { + const adapter = new ReauditAdapter(); + const cancelStarted = deferred(); + const releaseCancel = deferred(); + adapter.cancelStarted = cancelStarted.resolve; + adapter.cancelGate = releaseCancel.promise; + const persistence = new TakeoverCompletionRacePorts(); + const initial = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + await initial.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const cancelling = initial.cancel({ ...control, reason: 'complete during takeover' }); + await cancelStarted.promise; + const replacementAdapter = new ReauditAdapter(); + replacementAdapter.cancelStarted = releaseCancel.resolve; + const replacement = new GoalSessionSupervisor(replacementAdapter, persistence.asRuntimePorts()); + const reopened = await replacement.openSession({ ...identity, provider: replacementAdapter.provider, controllerEpoch: 2 }); + assert.equal(reopened.status, 'terminated'); + assert.equal((await cancelling).status, 'terminated'); + assert.equal(replacementAdapter.openCalls, 0); + assert.equal(replacementAdapter.cancelCalls.length, 1, 'replay uses the same durable cancellation identity'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('next-turn model application is crash-safe at pre-call, post-provider/pre-CAS, and post-CAS windows', async t => { + const capabilities = { + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + } as const; + const recoverInvocation = async (adapter: ReauditAdapter, persistence: InMemoryGoalSessionPorts) => { + persistence.setContainerInspection(identity, { status: 'missing' }); + persistence.setRepositoryInspection(repository, { + ...repository, exists: true, observedRepository: repository.repository, + observedBranch: repository.branch, observedHeadSha: repository.headSha, + resolvedWorktreePath: repository.worktreePath, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcile = async () => ({ + outcome: 'resumed' as const, reason: 'replay exact deferred invocation', + snapshot: { providerSessionId: 'reaudit-native', recoveryMetadata: { checkpoint: 'recovered' }, model: 'model-a' }, + }); + const recovered = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const reconciled = await recovered.reconcile(identity, 2, repository); + assert.equal(reconciled.outcome, 'resumed'); + return recovered.resumeTurn({ ...identity, controllerEpoch: 2 }); + }; + await t.test('pre-call', async () => { + const adapter = new ReauditAdapter(capabilities); + const persistence = new InMemoryGoalSessionPorts(); + const { supervisor } = await openRuntime(adapter, persistence); + persistence.setTransitionFault('after_commit'); + await assert.rejects(supervisor.runTurn(turnRequest('model-b')), /after state\/audit transaction commit/); + assert.equal(adapter.turnCalls.length, 0); + const recovered = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + assert.equal((await recovered.runTurn(turnRequest('model-b'))).state.currentModel, 'model-b'); + assert.equal(adapter.turnModelEffects.size, 1); + }); + + await t.test('post-provider/pre-CAS', async () => { + const adapter = new ReauditAdapter(capabilities); + const { persistence, supervisor } = await openRuntime(adapter); + await supervisor.requestModelChange({ ...control, model: 'model-b' }); + persistence.setTransitionFault('before_commit'); + await assert.rejects(supervisor.runTurn(turnRequest()), /before state\/audit transaction commit/); + await recoverInvocation(adapter, persistence); + assert.equal(adapter.turnCalls.length, 2); + assert.equal(new Set(adapter.turnCalls.map(call => call.modelChange?.modelChangeId)).size, 1); + assert.equal(adapter.turnModelEffects.size, 1); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); + }); + + await t.test('post-CAS', async () => { + const adapter = new ReauditAdapter(capabilities); + const { persistence, supervisor } = await openRuntime(adapter); + await supervisor.requestModelChange({ ...control, model: 'model-b' }); + persistence.setTransitionFault('after_commit'); + await assert.rejects(supervisor.runTurn(turnRequest()), /after state\/audit transaction commit/); + const applied = await persistence.load(identity); + assert.equal(applied?.currentModel, 'model-b'); + assert.equal(applied?.modelChangeIntent?.model, 'model-b', 'the applied claim remains until the turn is durably claimed'); + await recoverInvocation(adapter, persistence); + assert.equal(adapter.turnCalls.length, 2); + assert.equal(adapter.turnModelEffects.size, 1); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'model_changed').length, 1); + }); +}); + +class TerminalRacePorts extends InMemoryGoalSessionPorts { + beforeFirstTurnTerminal: (() => Promise) | undefined; + + override async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ) { + if ('scope' in operation && operation.scope === 'turn' && this.beforeFirstTurnTerminal) { + const hook = this.beforeFirstTurnTerminal; + this.beforeFirstTurnTerminal = undefined; + await hook(); + } + return super.commit(expected, next, operation); + } +} + +test('final-load to terminal-commit pause race retries exact attempt into canonical boundary and completion', async () => { + const adapter = new ReauditAdapter({ + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + }); + const persistence = new TerminalRacePorts(); + const { supervisor } = await openRuntime(adapter, persistence); + persistence.beforeFirstTurnTerminal = async () => { + await supervisor.requestPause({ ...control, reason: 'wins after final load' }); + }; + const result = await supervisor.runTurn(turnRequest()); + assert.equal(result.state.status, 'paused'); + const canonical = (await persistence.replay(identity)).filter(record => + record.event.type === 'pause_requested' + || record.event.type === 'pause_boundary' + || record.event.type === 'completion'); + assert.deepEqual(canonical.map(record => record.event.type), [ + 'pause_requested', 'pause_boundary', 'completion', + ]); + assert.equal(canonical.filter(record => record.event.type === 'completion').length, 1); + assert.equal((await supervisor.runTurn(turnRequest())).disposition, 'duplicate'); +}); + +class CancelBeforeAuditPorts extends InMemoryGoalSessionPorts { + beforeAudit: (() => Promise) | undefined; + + override async commit( + expected: GoalSessionState, + next: Omit, + operation: GoalTerminalCommit | GoalSessionControlTransition, + ) { + if (!('scope' in operation) && this.beforeAudit) { + const hook = this.beforeAudit; + this.beforeAudit = undefined; + await hook(); + } + return super.commit(expected, next, operation); + } +} + +test('atomic model/pause audits never append after a same-epoch terminal cancellation', async t => { + for (const operation of ['model', 'pause'] as const) { + await t.test(operation, async () => { + const adapter = new ReauditAdapter({ + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + }); + const persistence = new CancelBeforeAuditPorts(); + const { supervisor } = await openRuntime(adapter, persistence); + persistence.beforeAudit = async () => { + await supervisor.cancel({ ...control, reason: 'terminal wins before audit transaction' }); + }; + const pending = operation === 'model' + ? supervisor.requestModelChange({ ...control, model: 'model-b' }) + : supervisor.requestPause({ ...control, reason: 'too late' }); + await assert.rejects(pending, StaleGoalSessionFenceError); + const events = await persistence.replay(identity); + assert.deepEqual(events.map(record => record.event.type), ['completion']); + assert.equal(events.some(record => record.event.type === 'model_change_acknowledged' + || record.event.type === 'model_changed' + || record.event.type === 'pause_boundary'), false); + }); + } +}); + +test('atomic audit crash windows persist both state and event or neither before terminal ordering', async t => { + for (const fault of ['before_commit', 'after_commit'] as const) { + await t.test(fault, async () => { + const adapter = new ReauditAdapter({ + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + }); + const { persistence, supervisor } = await openRuntime(adapter); + persistence.setTransitionFault(fault); + await assert.rejects(supervisor.requestModelChange({ ...control, model: 'model-b' }), /state\/audit transaction commit/); + const afterFault = await persistence.load(identity); + const beforeCommit = fault === 'before_commit'; + assert.equal(afterFault?.pendingModelChange, beforeCommit ? undefined : 'model-b'); + assert.equal((await persistence.replay(identity)).some(record => + record.event.type === 'model_change_acknowledged'), !beforeCommit); + await supervisor.requestModelChange({ ...control, model: 'model-b' }); + assert.equal((await persistence.replay(identity)).filter(record => + record.event.type === 'model_change_acknowledged').length, 1); + await supervisor.cancel({ ...control, reason: 'finish after audit crash window' }); + const types = (await persistence.replay(identity)).map(record => record.event.type); + assert.equal(types.at(-1), 'completion'); + assert.equal(types.slice(types.indexOf('completion') + 1).some(type => + type === 'model_change_acknowledged' || type === 'model_changed' + || type === 'pause_boundary'), false); + }); + } +}); diff --git a/packages/core/test/goalSessionRecovery.test.ts b/packages/core/test/goalSessionRecovery.test.ts new file mode 100644 index 000000000..f57f7f724 --- /dev/null +++ b/packages/core/test/goalSessionRecovery.test.ts @@ -0,0 +1,100 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { DockerGoalSessionRecovery } from '../src/agents/goalSession/DockerGoalSessionRecovery.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; + +const gitPath = '/usr/bin/git'; + +function createRepository(remote = 'https://github.com/foreign/replacement.git'): { root: string; head: string } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'goal-recovery-repository-')); + execFileSync(gitPath, ['init', '--initial-branch=actual-branch', root]); + execFileSync(gitPath, ['config', 'user.email', 'goal-test@example.invalid'], { cwd: root }); + execFileSync(gitPath, ['config', 'user.name', 'Goal Test'], { cwd: root }); + fs.writeFileSync(path.join(root, 'checkpoint.txt'), 'authoritative checkout\n'); + execFileSync(gitPath, ['add', 'checkpoint.txt'], { cwd: root }); + execFileSync(gitPath, ['commit', '-m', 'authoritative checkout'], { cwd: root }); + execFileSync(gitPath, ['remote', 'add', 'origin', remote], { cwd: root }); + const head = execFileSync(gitPath, ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + return { root, head }; +} + +test('repository recovery observes origin, branch, head, and root from Git instead of expected request values', async () => { + const { root, head } = createRepository(); + const recovery = new DockerGoalSessionRecovery('/bin/false', gitPath); + const expected = { + repository: 'integry/propr', + worktreePath: root, + branch: 'expected-branch', + headSha: 'expected-head', + }; + const inspection = await recovery.inspectRepository(expected); + + assert.equal(inspection.exists, true); + assert.equal(inspection.observedRepository, 'foreign/replacement'); + assert.equal(inspection.observedBranch, 'actual-branch'); + assert.equal(inspection.observedHeadSha, head); + assert.equal(inspection.resolvedWorktreePath, root); + assert.equal(inspection.observedWorktreeFingerprint, fingerprintGoalWorktree({ + repository: 'foreign/replacement', + worktreePath: root, + branch: 'actual-branch', + headSha: head, + })); + assert.notEqual(inspection.observedWorktreeFingerprint, fingerprintGoalWorktree(expected)); +}); + +test('repository recovery refuses a path alias instead of reporting expected-derived checkout identity', async () => { + const { root } = createRepository(); + const alias = `${root}-alias`; + fs.symlinkSync(root, alias); + const recovery = new DockerGoalSessionRecovery('/bin/false', gitPath); + const inspection = await recovery.inspectRepository({ + repository: 'integry/propr', + worktreePath: alias, + branch: 'actual-branch', + headSha: 'not-the-observed-head', + }); + + assert.equal(inspection.exists, true); + assert.equal(inspection.resolvedWorktreePath, root); + assert.match(inspection.reason ?? '', /symlink or alias/); + assert.equal(inspection.observedWorktreeFingerprint, undefined); +}); + +test('Docker recovery rejects credential-bearing HTTPS and SSH/scp remotes without returning secrets', async t => { + const secret = 'DOCKER-REMOTE-SECRET'; + for (const [name, remote] of [ + ['HTTPS', `https://automation:${secret}@github.com/integry/propr.git`], + ['SSH URL', `ssh://git:${secret}@github.com/integry/propr.git`], + ['scp', `${secret}@github.com:integry/propr.git`], + ]) { + await t.test(name, async () => { + const { root } = createRepository(remote); + const recovery = new DockerGoalSessionRecovery('/bin/false', gitPath); + const inspection = await recovery.inspectRepository({ + repository: 'integry/propr', worktreePath: root, branch: 'actual-branch', + }); + assert.equal(inspection.observedRepository, undefined); + assert.match(inspection.reason ?? '', /trustworthy repository identity/); + assert.doesNotMatch(JSON.stringify(inspection), new RegExp(secret)); + }); + } +}); + +test('Docker recovery fails closed with a generic credential-free result for malformed remotes', async () => { + const secret = 'MALFORMED-REMOTE-SECRET'; + const { root } = createRepository(`https://automation:${secret}@/integry/propr`); + const recovery = new DockerGoalSessionRecovery('/bin/false', gitPath); + const inspection = await recovery.inspectRepository({ + repository: 'integry/propr', worktreePath: root, branch: 'actual-branch', + }); + + assert.equal(inspection.observedRepository, undefined); + assert.equal(inspection.observedWorktreeFingerprint, undefined); + assert.match(inspection.reason ?? '', /trustworthy repository identity/); + assert.doesNotMatch(JSON.stringify(inspection), new RegExp(secret)); +}); diff --git a/packages/core/test/goalSessionRuntimeFoundationAudit.test.ts b/packages/core/test/goalSessionRuntimeFoundationAudit.test.ts new file mode 100644 index 000000000..0845f2591 --- /dev/null +++ b/packages/core/test/goalSessionRuntimeFoundationAudit.test.ts @@ -0,0 +1,291 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import Database from 'better-sqlite3'; +import type { + GoalBeginTurnRequest, GoalProviderBarrierPublication, GoalProviderModelChangeRequest, GoalProviderOpenRequest, + GoalProviderSessionSnapshot, GoalSessionAdapter, GoalSessionEvent, +} from '../src/agents/goalSession/contract.js'; +import { GoalSessionSupervisor } from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { GoalSessionContractError } from '../src/agents/goalSession/errors.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { sanitizeRecoveryMetadata } from '../src/agents/goalSession/recoveryMetadata.js'; +import { sanitizeGoalSessionEvent } from '../src/agents/goalSession/securityBoundary.js'; +import { isSensitiveHostSourcePath } from '../src/agents/goalSession/worktreeIdentity.js'; +import { SqliteGoalSessionTestPorts } from './SqliteGoalSessionTestPorts.js'; + +const identity = { goalId: 'foundation-audit-goal', sessionId: 'foundation-audit-session' }; + +class IdentityAuditAdapter implements GoalSessionAdapter { + readonly provider = 'identity-audit'; + readonly capabilities: GoalSessionAdapter['capabilities']; + readonly traces: string[] = []; + openedPersisted?: GoalProviderOpenRequest['persisted']; + + constructor(modelChange: 'next_safe_boundary' | 'next_turn') { + this.capabilities = { + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange, + }; + } + + async publishOperationBarrier(_publication: GoalProviderBarrierPublication): Promise {} + async openSession(request: GoalProviderOpenRequest): Promise { + this.openedPersisted = request.persisted; + return { providerSessionId: 'identity-audit-native', recoveryMetadata: {}, model: 'model-0' }; + } + async *beginTurn(_request: GoalBeginTurnRequest): AsyncIterable { + yield { type: 'completion', outcome: 'succeeded' }; + } + async resumeSession(_request: never, snapshot: GoalProviderSessionSnapshot) { return snapshot; } + async requestModelChange(request: GoalProviderModelChangeRequest) { + this.traces.push(request.modelChangeId); + return { requestedModel: request.model, appliesAt: 'next_safe_boundary' as const, effectiveModel: request.model }; + } + async cancel() {} + async reconcile(): Promise<{ outcome: 'failed'; reason: string }> { return { outcome: 'failed', reason: 'not used' }; } +} + +test('event and recovery codecs reject traversal, endpoints, commands, extras, and invalid numbers', () => { + for (const file of [ + '.', '..', '../secret', 'src/../secret', '/etc/passwd', 'C:\\secret', '\\\\server\\share', + 'file:///tmp/a', 'docker://engine', 'src/a.ts;docker ps', 'src//a.ts', `src/${'a'.repeat(1025)}`, + ]) { + assert.throws(() => sanitizeGoalSessionEvent({ + type: 'tool', toolCallId: 'tool-1', name: 'read', phase: 'completed', data: { file }, + })); + } + assert.deepEqual(sanitizeGoalSessionEvent({ + type: 'tool', toolCallId: 'tool-1', name: 'read', phase: 'completed', data: { file: 'src/a.ts', line: 0 }, + }), { type: 'tool', toolCallId: 'tool-1', name: 'read', phase: 'completed', data: { file: 'src/a.ts', line: 0 } }); + for (const invalid of [Number.NaN, Number.POSITIVE_INFINITY, -1, 1.5]) { + assert.throws(() => sanitizeGoalSessionEvent({ type: 'usage', occurrenceId: 'usage-invalid', semantics: 'delta', watermark: 0, inputTokens: invalid })); + assert.throws(() => sanitizeRecoveryMetadata({ offset: invalid })); + } + for (const poisoned of [ + { checkpoint: '../escape' }, { checkpoint: 'file:///tmp/a' }, { offset: -1 }, + { command: 'docker ps' }, { checkpoint: 'ok', nested: { token: 'not-forwarded' } }, + ]) assert.throws(() => sanitizeRecoveryMetadata(poisoned)); +}); + +test('both model capability profiles reject unsafe caller IDs before history or provider traces', async () => { + for (const profile of ['next_safe_boundary', 'next_turn'] as const) { + const adapter = new IdentityAuditAdapter(profile); + const ports = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const beforeState = await ports.load(identity); + const beforeEvents = await ports.replay(identity); + for (const operationId of [ + '../operation', '/tmp/operation', 'C:\\operation', '\\\\server\\operation', + 'file:///tmp/operation', 'docker://engine', 'operation;docker ps', `x${'a'.repeat(256)}`, + ]) { + await assert.rejects(supervisor.requestModelChange({ + ...identity, controllerEpoch: 1, model: 'model-safe', operationId, + })); + } + await assert.rejects(supervisor.requestModelChange({ + ...identity, controllerEpoch: 1, model: '../model', operationId: 'safe-operation', + })); + assert.deepEqual(await ports.load(identity), beforeState); + assert.deepEqual(await ports.replay(identity), beforeEvents); + assert.deepEqual(adapter.traces, []); + } +}); + +test('reopen rejects durable poison without mutation and provider URL exceptions cross as generic errors', async () => { + const timestamp = new Date().toISOString(); + const ports = new InMemoryGoalSessionPorts(); + await ports.create({ + ...identity, provider: 'identity-audit', providerSessionId: 'identity-audit-native', + recoveryMetadata: { checkpoint: 'safe', command: 'docker ps', nested: { token: 'opaque-value' } }, + controllerEpoch: 1, status: 'idle', currentModel: 'model-0', completedTurnIds: [], + createdAt: timestamp, updatedAt: timestamp, legacyEnvelope: { command: 'docker ps', credential: 'opaque-value' }, + }); + const adapter = new IdentityAuditAdapter('next_turn'); + const poisonedBefore = await ports.load(identity); + await assert.rejects(new GoalSessionSupervisor(adapter, ports.asRuntimePorts()).openSession({ + ...identity, provider: adapter.provider, controllerEpoch: 2, + }), (error: unknown) => error instanceof GoalSessionContractError && error.code === 'INVALID_DURABLE_STATE'); + assert.deepEqual(await ports.load(identity), poisonedBefore); + assert.equal(adapter.openedPersisted, undefined); + + class ThrowingAdapter extends IdentityAuditAdapter { + override async openSession(): Promise { + throw new Error('request failed https://user:TOP-SECRET-CREDENTIAL@example.test/api command docker run'); + } + } + const poisonedPorts = new InMemoryGoalSessionPorts(); + const throwing = new ThrowingAdapter('next_turn'); + const failure = await new GoalSessionSupervisor(throwing, poisonedPorts.asRuntimePorts()).openSession({ + goalId: 'provider-error-goal', sessionId: 'provider-error-session', provider: throwing.provider, controllerEpoch: 1, + }).catch(error => error as Error); + assert.equal(failure.message, 'Provider operation failed safely'); + assert.doesNotMatch(JSON.stringify(failure), /TOP-SECRET|example\.test|docker run/); +}); + +test('host source policy blocks system, credential, and engine state while preserving project roots', () => { + for (const source of [ + '/run', '/run/docker.sock', '/var/run', '/var/run/docker.sock', '/proc/self/environ', + '/etc/passwd', '/root/arbitrary-file', '/boot/grub', '/var/lib/docker/overlay2', + '/var/lib/containers/storage', '/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs', + ]) assert.equal(isSensitiveHostSourcePath(source), true, source); + assert.equal(isSensitiveHostSourcePath('/var/www/project'), false); + assert.equal(isSensitiveHostSourcePath('/usr/src/project'), false); +}); + +test('supervisor cancellation claim wins at the process-like adapter first-effect transaction', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'provider-barrier-audit-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const filename = path.join(directory, 'provider.sqlite'); + const supervisorPorts = new SqliteGoalSessionTestPorts(filename); + const adapterPorts = new SqliteGoalSessionTestPorts(filename); + t.after(() => { supervisorPorts.close(); adapterPorts.close(); }); + let releaseTurnPublication!: () => void; + let turnPublicationStarted!: () => void; + let releaseCancellation!: () => void; + const turnPublicationGate = new Promise(resolve => { releaseTurnPublication = resolve; }); + const publicationStarted = new Promise(resolve => { turnPublicationStarted = resolve; }); + const cancellationGate = new Promise(resolve => { releaseCancellation = resolve; }); + class ProcessLikeAdapter extends IdentityAuditAdapter { + override async publishOperationBarrier(publication: GoalProviderBarrierPublication): Promise { + if (publication.pendingCancellationId) await cancellationGate; + else if (publication.generation > 1) { + turnPublicationStarted(); + await turnPublicationGate; + } + } + override async *beginTurn(request: GoalBeginTurnRequest): AsyncIterable { + void request; + yield { type: 'completion', outcome: 'succeeded' }; + } + } + const adapter = new ProcessLikeAdapter('next_turn'); + const first = new GoalSessionSupervisor(adapter, supervisorPorts.asRuntimePorts()); + await first.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const effectsBeforeTurn = adapterPorts.providerEffectCount(); + const running = first.runTurn({ + ...identity, controllerEpoch: 1, turnId: 'barrier-turn', executionId: 'barrier-execution', + attemptId: 'barrier-attempt', objective: 'prove exact provider boundary', + repository: { repository: 'integry/propr', worktreePath: '/tmp/provider-boundary', branch: 'audit' }, + requestedModel: 'model-0', + }); + await publicationStarted; + const cancelling = new GoalSessionSupervisor(adapter, adapterPorts.asRuntimePorts()).cancel({ + ...identity, controllerEpoch: 1, reason: 'invalidate before first effect', + }); + for (let attempt = 0; attempt < 100; attempt += 1) { + if ((await supervisorPorts.load(identity))?.providerBarrierIntent?.phase === 'pending') break; + await new Promise(resolve => setImmediate(resolve)); + } + const invalidated = (await supervisorPorts.load(identity))!; + releaseTurnPublication(); + await assert.rejects(running); + assert.equal(adapterPorts.providerEffectCount(), effectsBeforeTurn); + releaseCancellation(); + assert.equal((await cancelling).status, 'terminated'); +}); + +test('SQLite takeover settles one published cancellation barrier without replacing terminal ownership', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cancel-takeover-audit-')); + const filename = path.join(directory, 'state.sqlite'); + const firstPorts = new SqliteGoalSessionTestPorts(filename); + const secondPorts = new SqliteGoalSessionTestPorts(filename); + t.after(() => { + firstPorts.close(); secondPorts.close(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + let releaseCancel!: () => void; + const cancelGate = new Promise(resolve => { releaseCancel = resolve; }); + class GatedCancelAdapter extends IdentityAuditAdapter { + cancelCalls = 0; + override async cancel(): Promise { + this.cancelCalls += 1; + await cancelGate; + } + } + const adapter = new GatedCancelAdapter('next_turn'); + const first = new GoalSessionSupervisor(adapter, firstPorts.asRuntimePorts()); + const second = new GoalSessionSupervisor(adapter, secondPorts.asRuntimePorts()); + await first.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const cancelling = first.cancel({ ...identity, controllerEpoch: 1, reason: 'gated published cancellation' }); + for (let attempt = 0; attempt < 100 && adapter.cancelCalls < 1; attempt += 1) { + await new Promise(resolve => setImmediate(resolve)); + } + const takeover = second.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 2 }); + const published = await secondPorts.load(identity); + assert.ok(published?.status === 'cancelling' || published?.status === 'terminated'); + assert.equal(published?.controllerEpoch, 1); + const cancellationId = published?.cancellationIntent?.cancellationId; + assert.equal(adapter.cancelCalls, 1, 'takeover adopts the durable stage without a second provider call'); + + releaseCancel(); + const [cancelled, reopened] = await Promise.all([cancelling, takeover]); + assert.equal(cancelled.status, 'terminated'); + assert.equal(reopened.status, 'terminated'); + assert.equal(reopened.controllerEpoch, 1, 'terminal takeover never replaces the cancellation owner'); + assert.equal(reopened.cancellationIntent?.cancellationId, cancellationId); + assert.equal((await secondPorts.replay(identity)).filter(record => record.event.type === 'completion').length, 1); + const calls = adapter.cancelCalls; + const terminalRace = await second.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 3 }); + assert.equal(terminalRace.controllerEpoch, 1); + assert.equal(adapter.cancelCalls, calls, 'terminal takeover performs no provider mutation'); +}); + +test('independent processes allocate unique exact model order and deterministically retain newest 64', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'model-order-audit-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const filename = path.join(directory, 'state.sqlite'); + const seed = new SqliteGoalSessionTestPorts(filename); + seed.close(); + const moduleUrl = new URL('./SqliteGoalSessionTestPorts.ts', import.meta.url).href; + const lock = new Database(filename); + lock.exec('BEGIN IMMEDIATE'); + const busyRetry = runChildProcess(` + import { SqliteGoalSessionTestPorts } from ${JSON.stringify(moduleUrl)}; + const ports = new SqliteGoalSessionTestPorts(${JSON.stringify(filename)}); + const identity = ${JSON.stringify(identity)}; + await ports.claim(identity, 'busy-operation', 'model-busy'); + await ports.settle(identity, 'busy-operation', { requestedModel: 'model-busy', appliesAt: 'next_turn' }); + ports.close(); + `); + await new Promise(resolve => setTimeout(resolve, 100)); + lock.exec('COMMIT'); + lock.close(); + await busyRetry; + await Promise.all(Array.from({ length: 4 }, (_, worker) => runChildProcess(` + import { SqliteGoalSessionTestPorts } from ${JSON.stringify(moduleUrl)}; + const ports = new SqliteGoalSessionTestPorts(${JSON.stringify(filename)}); + const identity = ${JSON.stringify(identity)}; + for (let index = 0; index < 25; index += 1) { + const id = 'worker-' + ${JSON.stringify(worker)} + '-' + index; + await ports.claim(identity, id, 'model-' + id); + await ports.settle(identity, id, { requestedModel: 'model-' + id, appliesAt: 'next_turn' }); + } + ports.close(); + `))); + const database = new Database(filename, { readonly: true }); + t.after(() => database.close()); + const rows = database.prepare( + 'SELECT operation_id, sequence, status FROM goal_session_runtime_model_changes ORDER BY sequence', + ).all() as Array<{ operation_id: string; sequence: number; status: string }>; + assert.equal(rows.length, 101); + assert.equal(new Set(rows.map(row => row.sequence)).size, 101); + assert.deepEqual(rows.map(row => row.sequence), Array.from({ length: 101 }, (_, index) => index + 1)); + assert.equal(rows.filter(row => row.status === 'settled').length, 64); + assert.equal(rows.filter(row => row.status === 'retired').length, 37); +}); + +function runChildProcess(source: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--import', 'tsx', '--input-type=module', '--eval', source], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', chunk => { stderr += String(chunk); }); + child.once('error', reject); + child.once('exit', code => code === 0 ? resolve() : reject(new Error(stderr || `worker exited ${code}`))); + }); +} diff --git a/packages/core/test/goalSessionSevenBlocker.test.ts b/packages/core/test/goalSessionSevenBlocker.test.ts new file mode 100644 index 000000000..8e1ad0bee --- /dev/null +++ b/packages/core/test/goalSessionSevenBlocker.test.ts @@ -0,0 +1,341 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, GoalProviderModelChangeRequest, GoalProviderReconcileRequest, + GoalProviderResumeRequest, GoalProviderSessionSnapshot, GoalSessionAdapter, + GoalSessionControlFence, GoalSessionEvent, GoalSessionState, GoalSteeringRequest, +} from '../src/agents/goalSession/contract.js'; +import { GoalContainerSupervisor } from '../src/agents/goalSession/GoalContainerSupervisor.js'; +import { GoalSessionSupervisor } from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { InMemoryGoalSessionPorts } from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { MODEL_CHANGE_SETTLED_RETRY_HORIZON } from '../src/agents/goalSession/modelChangeProtocol.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; +import { SqliteGoalSessionTestPorts } from './SqliteGoalSessionTestPorts.js'; + +const identity = { goalId: 'seven-blocker-goal', sessionId: 'seven-blocker-session' }; +const control = { ...identity, controllerEpoch: 1 }; +const repository = { + repository: 'integry/propr', worktreePath: '/tmp/seven-blocker-worktree', branch: 'follow-up', +}; + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +class MatrixAdapter implements GoalSessionAdapter { + private barrierGeneration = 0; + async publishOperationBarrier(publication: { generation: number }): Promise { + this.barrierGeneration = Math.max(this.barrierGeneration, publication.generation); + } + protected assertProviderFence(generation: number): void { + if (generation < this.barrierGeneration) throw new Error('Provider operation was cancelled or replaced'); + } + readonly provider = 'matrix'; + readonly capabilities = { + nativeSessionId: 'eager' as const, steering: 'active_turn' as const, + pause: 'active_turn' as const, modelChange: 'next_safe_boundary' as const, + }; + reconcileRequests: GoalProviderReconcileRequest[] = []; + resumeRequests: GoalProviderResumeRequest[] = []; + modelRequests: GoalProviderModelChangeRequest[] = []; + reconcileOutcome: 'failed' | 'resumed' | 'alive' = 'resumed'; + resumeGate?: Promise; + + async openSession(): Promise { + return { providerSessionId: 'matrix-native', recoveryMetadata: {}, model: 'model-0' }; + } + async *beginTurn(_request: GoalBeginTurnRequest): AsyncIterable { + yield { type: 'completion', outcome: 'succeeded' }; + } + async *resumeTurn(): AsyncIterable { + yield { type: 'completion', outcome: 'succeeded' }; + } + async deliverMessage(request: { messageId: string }) { return { messageId: request.messageId }; } + async resumeSession(request: GoalProviderResumeRequest, snapshot: GoalProviderSessionSnapshot) { + this.resumeRequests.push(structuredClone(request)); + await this.resumeGate; + return snapshot; + } + async requestModelChange(request: GoalProviderModelChangeRequest) { + this.modelRequests.push(structuredClone(request)); + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + async requestPause() { return { appliesAt: 'next_safe_boundary' as const }; } + async cancel() {} + async reconcile(request: GoalProviderReconcileRequest) { + this.reconcileRequests.push(structuredClone(request)); + return this.reconcileOutcome === 'failed' + ? { outcome: 'failed' as const, reason: 'authoritative recovery failure' } + : this.reconcileOutcome === 'alive' + ? { outcome: 'alive' as const, reason: 'still alive' } + : { outcome: 'resumed' as const, reason: 'replaced', snapshot: { + providerSessionId: 'matrix-native', recoveryMetadata: {}, model: 'model-0', + } }; + } +} + +class GuardedSteeringAdapter extends MatrixAdapter { + readonly entered = deferred(); + readonly release = deferred(); + effects = 0; + + override async deliverMessage(request: GoalSteeringRequest) { + this.entered.resolve(); + await this.release.promise; + this.assertProviderFence(request.operationFence.generation); + this.effects += 1; + return { messageId: request.messageId }; + } +} + +function runningState(overrides: Partial = {}): Omit { + const timestamp = new Date().toISOString(); + return { + ...identity, provider: 'matrix', providerSessionId: 'matrix-native', recoveryMetadata: {}, + controllerEpoch: 1, status: 'running', currentModel: 'model-0', completedTurnIds: [], + activeTurn: { + turnId: 'turn-1', executionId: 'execution-1', attemptId: 'attempt-1', executionEpoch: 1, + objective: 'adversarial matrix', requestedModel: 'model-0', repository, status: 'running', + }, + createdAt: timestamp, updatedAt: timestamp, ...overrides, + }; +} + +function configureRecovery(ports: SqliteGoalSessionTestPorts | InMemoryGoalSessionPorts): void { + ports.setContainerInspection(identity, { status: 'missing' }); + ports.setRepositoryInspection(repository, { + ...repository, exists: true, observedRepository: repository.repository, + observedBranch: repository.branch, resolvedWorktreePath: repository.worktreePath, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); +} + +test('failed reconciliation atomically terminates every obligation and never repairs a pending model', async () => { + const ports = new InMemoryGoalSessionPorts(); + const adapter = new MatrixAdapter(); + adapter.reconcileOutcome = 'failed'; + const pendingModel = { + modelChangeId: 'pending-model', model: 'model-1', requestedAt: new Date().toISOString(), + generation: 1, phase: 'provider_in_doubt' as const, applicationToken: 'model-lease', + applicationControllerEpoch: 1, leaseExpiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + await ports.create(runningState({ + modelChangeGeneration: 1, + modelChangeIntent: pendingModel, + modelChangeIntents: [pendingModel], + })); + configureRecovery(ports); + const result = await new GoalSessionSupervisor(adapter, ports.asRuntimePorts()).reconcile(identity, 1, repository); + assert.equal(result.outcome, 'failed'); + assert.equal(result.state.status, 'failed'); + assert.equal(result.state.activeTurn, undefined); + assert.equal(result.state.modelChangeIntents, undefined); + assert.equal(adapter.modelRequests.length, 0); + assert.deepEqual((await ports.replay(identity)).map(record => record.event.type), ['reconciliation', 'completion']); + await assert.rejects(new GoalSessionSupervisor(adapter, ports.asRuntimePorts()) + .requestModelChange({ ...control, model: 'model-2', operationId: 'after-failure' }), /failed/); +}); + +test('two SQLite supervisors share one exclusive active resume lease and cancellation fences the late result', async () => { + const filename = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'resume-matrix-')), 'state.sqlite'); + const firstPorts = new SqliteGoalSessionTestPorts(filename); + const secondPorts = new SqliteGoalSessionTestPorts(filename); + await firstPorts.create(runningState({ + status: 'paused', activeTurn: { ...runningState().activeTurn!, status: 'paused' }, + })); + const gate = deferred(); + const firstAdapter = new MatrixAdapter(); + firstAdapter.resumeGate = gate.promise; + const secondAdapter = new MatrixAdapter(); + const first = new GoalSessionSupervisor(firstAdapter, firstPorts.asRuntimePorts(), () => 'resume-attempt-1'); + const second = new GoalSessionSupervisor(secondAdapter, secondPorts.asRuntimePorts(), () => 'resume-attempt-2'); + const pending = first.resumeTurn(control); + while (firstAdapter.resumeRequests.length === 0) await new Promise(resolve => setImmediate(resolve)); + const request = firstAdapter.resumeRequests[0]; + assert.equal(request.operationPhase, 'provider_in_doubt'); + assert.ok(request.operationGeneration > 0); + await assert.rejects(second.resumeTurn(control), /durable resume lease/); + assert.equal(secondAdapter.resumeRequests.length, 0); + await second.cancel({ ...control, reason: 'cancel across resume boundary' }); + gate.resolve(); + await assert.rejects(pending, /preempted|stale|fence/i); + assert.equal((await secondPorts.load(identity))?.status, 'terminated'); + firstPorts.close(); secondPorts.close(); +}); + +test('caller model operation IDs retry retained entries and report a retired ID without provider work', async () => { + const ports = new InMemoryGoalSessionPorts(); + const adapter = new MatrixAdapter(); + await ports.create(runningState({ status: 'idle', activeTurn: undefined })); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + for (let index = 0; index < MODEL_CHANGE_SETTLED_RETRY_HORIZON + 8; index += 1) { + await supervisor.requestModelChange({ ...control, model: `model-${index + 1}`, operationId: `operation-${index + 1}` }); + } + const state = await ports.load(identity); + assert.equal(state?.modelChangeIntents?.length, MODEL_CHANGE_SETTLED_RETRY_HORIZON); + const before = adapter.modelRequests.length; + const oldestRetired = await supervisor.requestModelChange({ ...control, model: 'model-1', operationId: 'operation-1' }); + assert.equal(oldestRetired.outcome, 'outside_retry_horizon'); + assert.equal(adapter.modelRequests.length, before); + const retained = await supervisor.requestModelChange({ + ...control, model: 'model-9', operationId: 'operation-9', + }); + assert.equal(retained.requestedModel, 'model-9'); + assert.equal(adapter.modelRequests.length, before); + const neverIssued = await supervisor.requestModelChange({ + ...control, model: 'model-adversarial-never-issued', operationId: 'adversarial-never-issued', + }); + assert.notEqual(neverIssued.outcome, 'outside_retry_horizon'); + assert.equal(adapter.modelRequests.length, before + 1); + await assert.rejects(supervisor.requestModelChange({ + ...control, model: 'model-conflict', operationId: 'adversarial-never-issued', + }), /different model/); + assert.ok(Buffer.byteLength(JSON.stringify(await ports.load(identity))) < 100_000); +}); + +test('next-turn model IDs stay exact across 5,001 issues and SQLite reopen', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'next-turn-model-matrix-')); + const filename = path.join(directory, 'state.sqlite'); + let ports = new SqliteGoalSessionTestPorts(filename); + t.after(() => { + ports.close(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + const adapter = new MatrixAdapter(); + Object.defineProperty(adapter, 'capabilities', { value: { + nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn', + } }); + await ports.create(runningState({ status: 'idle', activeTurn: undefined })); + let supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + for (let index = 0; index < 5_001; index += 1) { + await supervisor.requestModelChange({ ...control, model: `next-${index}`, operationId: `next-operation-${index}` }); + } + const bounded = await ports.load(identity); + assert.equal(bounded?.modelChangeIntent?.modelChangeId, 'next-operation-5000'); + assert.ok(Buffer.byteLength(JSON.stringify(bounded)) < 32_000); + ports.close(); + ports = new SqliteGoalSessionTestPorts(filename); + supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + assert.equal((await supervisor.requestModelChange({ + ...control, model: 'next-0', operationId: 'next-operation-0', + })).outcome, 'outside_retry_horizon'); + assert.equal((await supervisor.requestModelChange({ + ...control, model: 'next-4937', operationId: 'next-operation-4937', + })).requestedModel, 'next-4937'); + assert.notEqual((await supervisor.requestModelChange({ + ...control, model: 'never-issued', operationId: 'next-never-issued', + })).outcome, 'outside_retry_horizon'); + await assert.rejects(supervisor.requestModelChange({ + ...control, model: 'conflict', operationId: 'next-never-issued', + }), /different model/); +}); + +test('SQLite corrective-message consumption and acknowledgement event commit exactly once', async () => { + const filename = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'message-matrix-')), 'state.sqlite'); + const ports = new SqliteGoalSessionTestPorts(filename); + await ports.create(runningState()); + ports.enqueueMessage({ ...identity, messageId: 'message-1', sequence: 1, body: 'correct it', createdAt: new Date().toISOString() }); + const adapter = new MatrixAdapter(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + const outcome = await supervisor.deliverMessage({ ...control, turnId: 'turn-1', messageId: 'message-1', body: 'ignored poison' }); + assert.equal(outcome.acknowledgement, 'acknowledged'); + assert.equal((await ports.listPending(identity)).length, 0); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'message_acknowledged').length, 1); + const repeated = await supervisor.deliverMessage({ ...control, turnId: 'turn-1', messageId: 'message-1', body: 'repeat' }); + assert.equal(repeated.acknowledgement, 'already_acknowledged'); + assert.equal((await ports.replay(identity)).filter(record => record.event.type === 'message_acknowledged').length, 1); + ports.close(); +}); + +test('separate SQLite cancellation after the final steering read blocks provider consumption and ack', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'steering-guard-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const filename = path.join(directory, 'state.sqlite'); + const deliveryPorts = new SqliteGoalSessionTestPorts(filename); + const cancellationPorts = new SqliteGoalSessionTestPorts(filename); + t.after(() => { deliveryPorts.close(); cancellationPorts.close(); }); + await deliveryPorts.create(runningState()); + deliveryPorts.enqueueMessage({ + ...identity, messageId: 'guarded-message', sequence: 1, + body: 'consume only if current', createdAt: new Date().toISOString(), + }); + const adapter = new GuardedSteeringAdapter(); + const delivery = new GoalSessionSupervisor(adapter, deliveryPorts.asRuntimePorts()).deliverMessage({ + ...control, turnId: 'turn-1', executionId: 'execution-1', attemptId: 'attempt-1', + messageId: 'guarded-message', body: 'ignored', + }); + await adapter.entered.promise; + await new GoalSessionSupervisor(adapter, cancellationPorts.asRuntimePorts()).cancel({ + ...control, reason: 'cancel after final steering read', + }); + adapter.release.resolve(); + await assert.rejects(delivery, /Provider operation failed safely/); + assert.equal(adapter.effects, 0); + assert.equal((await deliveryPorts.listPending(identity)).length, 1); + assert.equal((await deliveryPorts.replay(identity)).filter(event => + event.event.type === 'message_acknowledged').length, 0); +}); + +test('hung provider cancellation is bounded and leaves one durable terminal completion', async () => { + const ports = new InMemoryGoalSessionPorts(); + await ports.create(runningState()); + const adapter = new MatrixAdapter(); + adapter.cancel = async () => new Promise(() => undefined); + const started = Date.now(); + const terminal = await new GoalSessionSupervisor(adapter, ports.asRuntimePorts()).cancel({ + ...control, reason: 'bounded cancellation', + }); + assert.equal(terminal.status, 'terminated'); + assert.ok(Date.now() - started < 2_000); + assert.equal((await ports.replay(identity)).filter(event => event.event.type === 'completion').length, 1); +}); + +test('credential poison is rejected before provider/state/event boundaries and sensitive allowlisted mounts still fail', async () => { + const ports = new InMemoryGoalSessionPorts(); + const adapter = new MatrixAdapter(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + await supervisor.openSession({ ...identity, provider: adapter.provider, controllerEpoch: 1 }); + const secret = 'ghp_1234567890SECRET'; + await assert.rejects(supervisor.runTurn({ + ...control, turnId: 'poison-turn', executionId: 'poison-execution', objective: 'poison', + repository: { ...repository, repository: `https://${secret}@github.com/integry/propr.git` }, + requestedModel: 'model-0', + }), /trustworthy Git repository/); + await assert.rejects(supervisor.runTurn({ + ...control, turnId: 'sensitive-turn', executionId: 'sensitive-execution', objective: 'poison', + repository: { ...repository, worktreePath: '/etc' }, requestedModel: 'model-0', + }), /trustworthy Git repository/); + const aliasRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'sensitive-alias-')); + const alias = path.join(aliasRoot, 'project'); + fs.symlinkSync('/etc', alias); + await assert.rejects(supervisor.runTurn({ + ...control, turnId: 'alias-turn', executionId: 'alias-execution', objective: 'poison', + repository: { ...repository, worktreePath: alias }, requestedModel: 'model-0', + }), /trustworthy Git repository/); + fs.rmSync(aliasRoot, { recursive: true, force: true }); + assert.equal(adapter.reconcileRequests.length, 0); + assert.doesNotMatch(JSON.stringify(await ports.load(identity)), new RegExp(secret)); + assert.doesNotMatch(JSON.stringify(await ports.replay(identity)), new RegExp(secret)); + + const events = ports.asRuntimePorts().events; + const container = new GoalContainerSupervisor('/tmp/seven-blocker-containers', events, undefined, { + isolation: { + environmentKeys: [], worktreePaths: ['/etc'], providerHomeTargets: ['/opt/provider'], credentialMounts: [], + }, + providerFirstEffects: ports.asRuntimePorts().providerFirstEffects, + }); + await assert.rejects(container.start({ + ...control, turnId: 'mount-turn', executionId: 'mount-execution', attemptId: 'mount-attempt', + operationFence: { + ...control, turnId: 'mount-turn', executionId: 'mount-execution', attemptId: 'mount-attempt', + generation: 1, operationId: 'mount-operation', kind: 'turn', + }, + image: 'unused', command: ['true'], worktreePath: '/etc', worktreeFingerprint: 'fingerprint', + providerHomeTarget: '/opt/provider', + }), /sensitive host root or descendant/); +}); diff --git a/packages/core/test/goalSessionSliceOneCorrection.test.ts b/packages/core/test/goalSessionSliceOneCorrection.test.ts new file mode 100644 index 000000000..cf8ca315e --- /dev/null +++ b/packages/core/test/goalSessionSliceOneCorrection.test.ts @@ -0,0 +1,531 @@ +import assert from 'node:assert/strict'; +import Database from 'better-sqlite3'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import vm from 'node:vm'; +import type { + GoalProviderOperationFence, GoalSessionAdapter, GoalSessionState, +} from '../src/agents/goalSession/contract.js'; +import { GoalSessionSupervisor } from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { AuthoritativeGoalSessionRuntimePorts } from '../src/agents/goalSession/AuthoritativeGoalSessionRuntimePorts.js'; +import { controlOperationId } from '../src/agents/goalSession/controlOperationIdentity.js'; +import { + assertStartedProviderEffect, providerFirstEffectStream, startedProviderEffect, +} from '../src/agents/goalSession/providerEffectProtocol.js'; +import { SqliteGoalSessionTestPorts } from './SqliteGoalSessionTestPorts.js'; + +const repository = { repository: 'integry/propr', worktreePath: '/tmp/slice-one', branch: 'slice-one' }; +const recovery = { + async inspectContainer() { return { status: 'missing' as const }; }, + async inspectRepository() { return { ...repository, exists: false }; }, +}; + +function baseState(sessionId: string): Omit { + const now = new Date().toISOString(); + return { + goalId: 'slice-one-goal', sessionId, provider: 'slice-provider', + providerSessionId: 'native-session', recoveryMetadata: {}, controllerEpoch: 1, + status: 'idle', currentModel: 'model-a', completedTurnIds: [], + providerOperationGeneration: 1, createdAt: now, updatedAt: now, + }; +} + +function liveTurn(sessionId: string, status: 'running' | 'pause_requested' = 'running') { + const state = baseState(sessionId); + return { + ...state, status, + activeTurn: { + turnId: 'turn-live', executionId: 'execution-live', attemptId: 'attempt-live', + executionEpoch: 1, objective: 'slice one', requestedModel: 'model-a', repository, + providerOperationGeneration: 1, status, + }, + } satisfies Omit; +} + +function operationCase(kind: GoalProviderOperationFence['kind'], sessionId: string): { + state: Omit; + fence: GoalProviderOperationFence; +} { + const identity = { goalId: 'slice-one-goal', sessionId, controllerEpoch: 1, generation: 1, kind }; + const future = new Date(Date.now() + 60_000).toISOString(); + if (kind === 'open') { + const state = { ...baseState(sessionId), status: 'initializing' as const, + providerSessionId: undefined, recoveryMetadata: undefined, + providerOpenAttemptId: 'open-attempt', providerOpenOperationGeneration: 1 }; + return { state, fence: { ...identity, operationId: 'open-attempt' } }; + } + if (kind === 'turn' || kind === 'steer') { + const state = liveTurn(sessionId); + return { state, fence: { ...identity, + operationId: kind === 'turn' ? 'turn-live:execution-live:attempt-live' : 'message-live', + turnId: 'turn-live', executionId: 'execution-live', attemptId: 'attempt-live' } }; + } + if (kind === 'pause') { + const state = liveTurn(sessionId, 'pause_requested'); + return { state, fence: { ...identity, operationId: controlOperationId('pause', { ...state, version: 1 }), + turnId: 'turn-live', executionId: 'execution-live', attemptId: 'attempt-live' } }; + } + if (kind === 'resume') { + const state = { ...baseState(sessionId), status: 'paused' as const, resumeIntent: { + executionId: 'execution-live', attemptId: 'resume-attempt', operationId: 'resume-live', + operationGeneration: 1, kind: 'after_turn' as const, controllerEpoch: 1, + claimedAt: new Date().toISOString(), leaseExpiresAt: future, phase: 'provider_in_doubt' as const, + } }; + return { state, fence: { ...identity, operationId: 'resume-live', leaseExpiresAt: future, + executionId: 'execution-live', attemptId: 'resume-attempt' } }; + } + if (kind === 'reconcile') { + const state = { ...baseState(sessionId), recoveryAttemptId: 'recovery-attempt', recoveryAttempt: { + operationToken: 'reconcile-live', operationGeneration: 1, executionId: 'execution-live', + attemptId: 'recovery-attempt', controllerEpoch: 1, sessionStatus: 'idle' as const, + claimedAt: new Date().toISOString(), leaseExpiresAt: future, phase: 'provider_in_doubt' as const, + } }; + return { state, fence: { ...identity, operationId: 'reconcile-live', leaseExpiresAt: future, + executionId: 'execution-live', attemptId: 'recovery-attempt' } }; + } + if (kind === 'model') { + const intent = { + modelChangeId: 'model-change', model: 'model-b', requestedAt: new Date().toISOString(), + generation: 1, phase: 'provider_in_doubt' as const, applicationToken: 'application-token', + applicationControllerEpoch: 1, leaseExpiresAt: future, + }; + const state = { ...baseState(sessionId), modelChangeGeneration: 1, + modelChangeIntent: intent, modelChangeIntents: [intent] }; + return { state, fence: { ...identity, operationId: 'model-change:application-token', leaseExpiresAt: future } }; + } + const cancellationIntent = { + cancellationId: 'cancel-live', reason: 'cancel', claimedAt: new Date().toISOString(), + }; + const state = { ...baseState(sessionId), status: 'cancelling' as const, cancellationIntent, + providerBarrierIntent: { generation: 1, operationId: 'cancel-live', kind: 'cancellation' as const, + phase: 'published' as const, claimedAt: cancellationIntent.claimedAt, pendingCancellationId: 'cancel-live' } }; + return { state, fence: { ...identity, operationId: 'cancel-live' } }; +} + +function invalidatedState( + state: GoalSessionState, + kind: GoalProviderOperationFence['kind'], +): Omit { + const { version: _version, ...current } = state; + if (kind === 'open' || kind === 'cancel') return { ...current, controllerEpoch: 2 }; + const claimedAt = new Date().toISOString(); + return { + ...current, status: 'cancelling', activeTurn: undefined, providerOperationGeneration: 2, + retryTurn: undefined, recoveryAttemptId: undefined, recoveryAttempt: undefined, + resumeIntent: undefined, completedResume: undefined, pendingAfterTurnPause: undefined, + modelChangeIntent: undefined, modelChangeIntents: undefined, + cancellationIntent: { cancellationId: `cancel-${kind}`, reason: 'cancel', claimedAt }, + providerBarrierIntent: { + generation: 2, operationId: `cancel-${kind}`, kind: 'cancellation', phase: 'pending', + claimedAt, pendingCancellationId: `cancel-${kind}`, + }, + }; +} + +test('production SQLite cancellation/takeover races leave every stale primitive effect-free', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-runtime-')); + const filename = path.join(directory, 'runtime.sqlite'); + const effectPorts = new SqliteGoalSessionTestPorts(filename); + const controllerPorts = new SqliteGoalSessionTestPorts(filename); + const effectGate = effectPorts.asRuntimePorts().providerFirstEffects; + t.after(() => { effectPorts.close(); controllerPorts.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + + for (const kind of ['open', 'turn', 'steer', 'pause', 'resume', 'model', 'reconcile', 'cancel'] as const) { + await t.test(kind, async () => { + const { state, fence } = operationCase(kind, `session-${kind}`); + await effectPorts.create(state); + const current = (await controllerPorts.load(state))!; + assert.ok(await controllerPorts.compareAndSet(current, invalidatedState(current, kind))); + let effects = 0; + await assert.rejects(effectGate.start(fence, 'provider_primitive', () => { + effects += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + })); + assert.equal(effects, 0); + }); + } + assert.equal(effectPorts.providerEffectCount(), 0); +}); + +test('stream creation and first next remain effect-free after independent cancellation', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-stream-')); + const filename = path.join(directory, 'runtime.sqlite'); + const effects = new SqliteGoalSessionTestPorts(filename); + const controller = new SqliteGoalSessionTestPorts(filename); + t.after(() => { effects.close(); controller.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + const { state, fence } = operationCase('turn', 'stream-session'); + await effects.create(state); + let created = 0, firstNext = 0; + const stream = providerFirstEffectStream(effects.asRuntimePorts().providerFirstEffects, fence, () => { + created += 1; + return { [Symbol.asyncIterator]: () => ({ + next: async () => { + firstNext += 1; + return { done: true, value: undefined }; + }, + return: async () => ({ done: true, value: undefined }), + }) }; + }); + const current = (await controller.load(state))!; + const claimedAt = new Date().toISOString(); + assert.ok(await controller.compareAndSet(current, { + ...state, controllerEpoch: 1, status: 'cancelling', activeTurn: undefined, + providerOperationGeneration: 2, + cancellationIntent: { cancellationId: 'stream-cancel', reason: 'cancel', claimedAt }, + providerBarrierIntent: { + generation: 2, operationId: 'stream-cancel', kind: 'cancellation', phase: 'pending', + claimedAt, pendingCancellationId: 'stream-cancel', + }, + })); + await assert.rejects(stream[Symbol.asyncIterator]().next()); + assert.deepEqual({ created, firstNext, durableEffects: effects.providerEffectCount() }, + { created: 0, firstNext: 0, durableEffects: 0 }); +}); + +test('SQLite commits after synchronous start, rejects async callback escape, and awaits completion outside', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-handle-')); + const filename = path.join(directory, 'runtime.sqlite'); + const effects = new SqliteGoalSessionTestPorts(filename); + const controller = new SqliteGoalSessionTestPorts(filename); + const effectGate = effects.asRuntimePorts().providerFirstEffects; + t.after(() => { effects.close(); controller.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + const { state, fence } = operationCase('open', 'handle-session'); + await effects.create(state); + await assert.rejects(effectGate.start(fence, 'provider_primitive', (async () => startedProviderEffect( + Promise.resolve(), () => undefined, + )) as never), + /synchronously return/); + let finish!: () => void; + const completion = new Promise(resolve => { finish = resolve; }); + const pending = effectGate.start(fence, 'container_spawn', () => startedProviderEffect(completion, () => undefined)); + const current = (await controller.load(state))!; + assert.ok(await controller.compareAndSet(current, { ...state, controllerEpoch: 2 }), + 'the authoritative transaction commits before completion is awaited'); + finish(); + await pending; +}); + +class IdentityAdapter implements GoalSessionAdapter { + readonly provider = 'slice-provider'; + readonly capabilities = { nativeSessionId: 'eager', steering: 'next_turn', pause: 'after_turn', modelChange: 'next_turn' } as const; + calls = 0; + async publishOperationBarrier() {} + async openSession() { this.calls += 1; return { providerSessionId: 'native-session', recoveryMetadata: {} }; } + async *beginTurn() { this.calls += 1; yield { type: 'completion' as const, outcome: 'succeeded' as const }; } + async resumeSession(_request: never, snapshot: never) { return snapshot; } + async requestModelChange(request: { model: string }) { return { requestedModel: request.model, appliesAt: 'next_turn' as const }; } + async cancel() {} + async reconcile() { return { outcome: 'failed' as const, reason: 'unused' }; } +} + +test('adversarial caller turn/execution/attempt IDs leave zero state, event, or provider mutation', async () => { + const adapter = new IdentityAdapter(); + const memory = (await import('../src/agents/goalSession/InMemoryGoalSessionPorts.js')).InMemoryGoalSessionPorts; + const ports = new memory(); + const supervisor = new GoalSessionSupervisor(adapter, ports.asRuntimePorts()); + const identity = { goalId: 'caller-id-goal', sessionId: 'caller-id-session', controllerEpoch: 1 }; + await supervisor.openSession({ ...identity, provider: adapter.provider }); + const before = await ports.load(identity), events = await ports.replay(identity), calls = adapter.calls; + const poison = ['../escape', 'contains space', 'github_pat_secret', 'ghp_secret', 'sk-secret', 'AKIASECRET', `x${'a'.repeat(256)}`]; + for (const value of poison) { + for (const field of ['turnId', 'executionId', 'attemptId'] as const) { + const request = { ...identity, turnId: 'safe-turn', executionId: 'safe-execution', attemptId: 'safe-attempt', + objective: 'safe', repository, requestedModel: 'model-a', [field]: value }; + await assert.rejects(supervisor.runTurn(request)); + } + } + assert.deepEqual(await ports.load(identity), before); + assert.deepEqual(await ports.replay(identity), events); + assert.equal(adapter.calls, calls); +}); + +test('authoritative composition is mandatory and owns no standalone schema', () => { + assert.throws( + () => new AuthoritativeGoalSessionRuntimePorts(undefined as never, recovery), + /authoritative transaction domain/, + ); +}); + +test('exact #2018 control event/message tables coexist in both initialization orders', async t => { + const eventColumns = ['id', 'goal_id', 'sequence', 'kind', 'event_type', 'payload_json', + 'idempotency_key', 'lease_epoch', 'created_at']; + const messageColumns = ['message_id', 'goal_id', 'sequence', 'body', 'predefined_kind', 'state', + 'delivered_at', 'acknowledged_at', 'delivery_attempts', 'last_error', 'idempotency_key', 'created_at']; + for (const ordering of ['control_first', 'runtime_first'] as const) { + await t.test(ordering, () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-schema-')); + const filename = path.join(directory, 'runtime.sqlite'); + let runtime: SqliteGoalSessionTestPorts | undefined; + let database: Database.Database | undefined; + try { + if (ordering === 'runtime_first') runtime = new SqliteGoalSessionTestPorts(filename); + database = new Database(filename); + createExactControlTables(database); + if (ordering === 'control_first') runtime = new SqliteGoalSessionTestPorts(filename); + assert.deepEqual(tableColumns(database, 'goal_events'), eventColumns); + assert.deepEqual(tableColumns(database, 'goal_messages'), messageColumns); + assert.ok(tableColumns(database, 'goal_session_runtime_events').includes('payload')); + assert.ok(tableColumns(database, 'goal_session_runtime_messages').includes('payload')); + } finally { + database?.close(); + runtime?.close(); + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + } +}); + +test('global session ownership rejects every foreign-goal runtime surface across connections', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-owner-')); + const filename = path.join(directory, 'runtime.sqlite'); + const owner = new SqliteGoalSessionTestPorts(filename); + const foreign = new SqliteGoalSessionTestPorts(filename); + t.after(() => { owner.close(); foreign.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + const created = (await owner.create(baseState('globally-owned-session')))! + const alien = { goalId: 'foreign-goal', sessionId: created.sessionId }; + const alienState = { ...baseState(created.sessionId), goalId: alien.goalId }; + await assert.rejects(foreign.create(alienState), /different goal/); + await assert.rejects(foreign.load(alien), /different goal/); + await assert.rejects(foreign.replay(alien), /different goal/); + await assert.rejects(foreign.listPending(alien), /different goal/); + await assert.rejects(foreign.claim(alien, 'foreign-model-op', 'model-b'), /different goal/); + await assert.rejects(foreign.commit( + { ...created, goalId: alien.goalId }, + alienState, + { + scope: 'control', fence: { ...alien, controllerEpoch: 1 }, + execution: { executionId: 'foreign-execution', attemptId: 'foreign-attempt' }, + auditEvents: [], event: { type: 'completion', outcome: 'failed', error: 'foreign' }, + }, + ), /different goal/); + const gate = foreign.asRuntimePorts().providerFirstEffects; + await assert.rejects(gate.start({ + ...alien, controllerEpoch: 1, generation: 1, kind: 'open', operationId: 'foreign-open', + }, 'provider_primitive', () => startedProviderEffect(Promise.resolve(), () => undefined)), /different goal/); +}); + +test('each claimed stage starts once and distinct inner stages remain legitimate', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-stages-')); + const filename = path.join(directory, 'runtime.sqlite'); + const first = new SqliteGoalSessionTestPorts(filename); + const duplicate = new SqliteGoalSessionTestPorts(filename); + t.after(() => { first.close(); duplicate.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + const { state, fence } = operationCase('open', 'stage-session'); + await first.create(state); + let finish!: () => void; + const completion = new Promise(resolve => { finish = resolve; }); + let providerStarts = 0, duplicateStarts = 0, containerStarts = 0; + const provider = first.asRuntimePorts().providerFirstEffects.start(fence, 'provider_primitive', () => { + providerStarts += 1; + return startedProviderEffect(completion, () => undefined); + }); + await assert.rejects(duplicate.asRuntimePorts().providerFirstEffects.start( + fence, 'provider_primitive', () => { + duplicateStarts += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + }, + ), /in doubt/); + await duplicate.asRuntimePorts().providerFirstEffects.start(fence, 'container_spawn', () => { + containerStarts += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + }); + assert.deepEqual({ providerStarts, duplicateStarts, containerStarts }, { + providerStarts: 1, duplicateStarts: 0, containerStarts: 1, + }); + finish(); + await provider; +}); + +test('post-start receipt/commit failures clean up and permanently fence retry', async t => { + for (const fault of ['receipt_write', 'commit'] as const) { + await t.test(fault, async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-failure-')); + const filename = path.join(directory, 'runtime.sqlite'); + const effects = new SqliteGoalSessionTestPorts(filename); + const retry = new SqliteGoalSessionTestPorts(filename); + try { + const { state, fence } = operationCase('open', `failure-${fault}`); + await effects.create(state); + effects.setProviderFault(fault); + let starts = 0, cleanups = 0; + await assert.rejects(effects.asRuntimePorts().providerFirstEffects.start( + fence, 'provider_primitive', () => { + starts += 1; + return startedProviderEffect(Promise.resolve(), () => { cleanups += 1; }); + }, + ), /failure/); + await assert.rejects(retry.asRuntimePorts().providerFirstEffects.start( + fence, 'provider_primitive', () => { + starts += 1; + return startedProviderEffect(Promise.resolve(), () => { cleanups += 1; }); + }, + ), /in doubt/); + assert.deepEqual({ starts, cleanups }, { starts: 1, cleanups: 1 }); + } finally { + effects.close(); retry.close(); fs.rmSync(directory, { recursive: true, force: true }); + } + }); + } +}); + +test('delayed completion rejection is observed after receipt failure even when cleanup also fails', async t => { + for (const cleanupFails of [false, true]) { + await t.test(cleanupFails ? 'cleanup_failure' : 'cleanup_success', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-delayed-rejection-')); + const filename = path.join(directory, 'runtime.sqlite'); + const ports = new SqliteGoalSessionTestPorts(filename); + const unhandled: unknown[] = []; + const observe = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', observe); + try { + const { state, fence } = operationCase('open', `delayed-${cleanupFails}`); + await ports.create(state); + ports.setProviderFault('receipt_write'); + let rejectCompletion!: (error: Error) => void; + const completion = new Promise((_resolve, reject) => { rejectCompletion = reject; }); + await assert.rejects(ports.asRuntimePorts().providerFirstEffects.start( + fence, 'provider_primitive', () => startedProviderEffect(completion, () => { + if (cleanupFails) throw new Error('cleanup failed'); + }), + ), cleanupFails ? /cleanup failed/ : /receipt-write failure/); + rejectCompletion(new Error('delayed process rejection')); + await new Promise(resolve => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(unhandled, []); + } finally { + process.off('unhandledRejection', observe); + ports.close(); + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + } +}); + +test('cleanup failure and synchronous throw remain durable in-doubt without reentry', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-cleanup-')); + const filename = path.join(directory, 'runtime.sqlite'); + const ports = new SqliteGoalSessionTestPorts(filename); + const peer = new SqliteGoalSessionTestPorts(filename); + t.after(() => { ports.close(); peer.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + const firstCase = operationCase('open', 'cleanup-failure'); + await ports.create(firstCase.state); + ports.setProviderFault('commit'); + let starts = 0, reentrantStarts = 0; + const gate = ports.asRuntimePorts().providerFirstEffects; + await assert.rejects(gate.start(firstCase.fence, 'provider_primitive', () => { + starts += 1; + return startedProviderEffect(Promise.resolve(), async () => { + await assert.rejects(peer.asRuntimePorts().providerFirstEffects.start( + firstCase.fence, 'provider_primitive', () => { + reentrantStarts += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + }, + ), /in doubt/); + throw new Error('cancel failed'); + }); + }), /cleanup failed/); + await assert.rejects(gate.start(firstCase.fence, 'provider_primitive', () => { + starts += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + }), /in doubt/); + + const thrownCase = operationCase('open', 'sync-throw'); + await ports.create(thrownCase.state); + let synchronousEntries = 0; + await assert.rejects(gate.start(thrownCase.fence, 'provider_primitive', () => { + synchronousEntries += 1; + throw new Error('synchronous start failure'); + }), /synchronous start failure/); + await assert.rejects(gate.start(thrownCase.fence, 'provider_primitive', () => { + synchronousEntries += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + }), /in doubt/); + assert.deepEqual({ starts, reentrantStarts, synchronousEntries }, { + starts: 1, reentrantStarts: 0, synchronousEntries: 1, + }); +}); + +test('handle validation failure invokes safe cleanup and cannot retry the started stage', async t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'slice-one-invalid-handle-')); + const filename = path.join(directory, 'runtime.sqlite'); + const ports = new SqliteGoalSessionTestPorts(filename); + const peer = new SqliteGoalSessionTestPorts(filename); + t.after(() => { ports.close(); peer.close(); fs.rmSync(directory, { recursive: true, force: true }); }); + const { state, fence } = operationCase('open', 'invalid-handle'); + await ports.create(state); + let starts = 0, cleanups = 0; + const malformed = Object.freeze(Object.assign(Object.create(null), { + completion: Promise.resolve(), + cleanup: Object.freeze({ kind: 'rollback_or_cancel', run: () => { cleanups += 1; } }), + })); + await assert.rejects(ports.asRuntimePorts().providerFirstEffects.start( + fence, 'provider_primitive', () => { starts += 1; return malformed; }, + ), /started-effect handle/); + await assert.rejects(peer.asRuntimePorts().providerFirstEffects.start( + fence, 'provider_primitive', () => { + starts += 1; + return startedProviderEffect(Promise.resolve(), () => undefined); + }, + ), /in doubt/); + assert.deepEqual({ starts, cleanups }, { starts: 1, cleanups: 1 }); +}); + +test('started handles reject thenables, accessors, cross-realm promises, and forgery without assimilation', () => { + let assimilations = 0, getterReads = 0; + const lazyThenable = { then() { assimilations += 1; } }; + const accessor = Object.create(null); + Object.defineProperty(accessor, 'completion', { + get() { getterReads += 1; throw new Error('getter must not run'); }, + }); + Object.defineProperty(accessor, 'cleanup', { value: Object.freeze({ + kind: 'rollback_or_cancel', run: () => undefined, + }) }); + const callableThen = Object.freeze({ + completion: Promise.resolve(), cleanup: Object.freeze({ kind: 'rollback_or_cancel', run: () => undefined }), + then() { assimilations += 1; }, + }); + const crossRealm = vm.runInNewContext('Promise.resolve(1)') as Promise; + const unbranded = Object.freeze(Object.assign(Object.create(null), { + completion: Promise.resolve(), cleanup: Object.freeze({ kind: 'rollback_or_cancel', run: () => undefined }), + })); + for (const hostile of [ + Object.freeze(Object.assign(Object.create(null), { + completion: lazyThenable, cleanup: Object.freeze({ kind: 'rollback_or_cancel', run: () => undefined }), + })), + accessor, + callableThen, + unbranded, + ]) assert.throws(() => assertStartedProviderEffect(hostile), /started-effect handle/); + assert.throws(() => startedProviderEffect(crossRealm, () => undefined), /native completion/); + assert.deepEqual({ assimilations, getterReads }, { assimilations: 0, getterReads: 0 }); + const exact = startedProviderEffect(Promise.resolve(1), () => undefined); + assert.ok(Object.isFrozen(exact)); + assert.equal('then' in exact, false); + assert.doesNotThrow(() => assertStartedProviderEffect(exact)); +}); + +function createExactControlTables(database: Database.Database): void { + database.exec(` + CREATE TABLE goal_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, goal_id TEXT NOT NULL, sequence INTEGER NOT NULL, + kind TEXT NOT NULL, event_type TEXT NOT NULL, payload_json TEXT, idempotency_key TEXT NOT NULL, + lease_epoch INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL + ); + CREATE UNIQUE INDEX goal_events_goal_sequence_idx ON goal_events(goal_id, sequence); + CREATE UNIQUE INDEX goal_events_goal_idempotency_idx ON goal_events(goal_id, idempotency_key); + CREATE TABLE goal_messages ( + message_id TEXT PRIMARY KEY, goal_id TEXT NOT NULL, sequence INTEGER NOT NULL, body TEXT NOT NULL, + predefined_kind TEXT, state TEXT NOT NULL DEFAULT 'queued', delivered_at TEXT, acknowledged_at TEXT, + delivery_attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT, idempotency_key TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE UNIQUE INDEX goal_messages_goal_sequence_idx ON goal_messages(goal_id, sequence); + CREATE UNIQUE INDEX goal_messages_goal_idempotency_idx ON goal_messages(goal_id, idempotency_key); + `); +} + +function tableColumns(database: Database.Database, table: string): string[] { + return (database.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(row => row.name); +} diff --git a/packages/core/test/goalSessionSupervisor.test.ts b/packages/core/test/goalSessionSupervisor.test.ts new file mode 100644 index 000000000..8f4477642 --- /dev/null +++ b/packages/core/test/goalSessionSupervisor.test.ts @@ -0,0 +1,950 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { + GoalBeginTurnRequest, + GoalCancelRequest, + GoalModelChangeRequest, + GoalPauseRequest, + GoalProviderOpenRequest, + GoalProviderReconcileRequest, + GoalProviderReconcileResult, + GoalProviderSessionSnapshot, + GoalSessionAdapter, + GoalSessionControlFence, + GoalSessionEvent, + GoalSessionFence, + GoalSteeringRequest, +} from '../src/agents/goalSession/contract.js'; +import { + GoalSessionContractError, + GoalSessionSupervisor, + StaleGoalSessionFenceError, + UnsupportedGoalSessionTransitionError, +} from '../src/agents/goalSession/GoalSessionSupervisor.js'; +import { + GoalSessionScopeError, + InMemoryGoalSessionPorts, +} from '../src/agents/goalSession/InMemoryGoalSessionPorts.js'; +import { fingerprintGoalWorktree } from '../src/agents/goalSession/worktreeIdentity.js'; + +const identity = { goalId: 'goal-2007', sessionId: 'session-one' }; +const repository = { + repository: 'integry/propr', + worktreePath: '/tmp/propr-goal-2007', + branch: 'goal-branch', + headSha: 'abc123', +}; +const fence: GoalSessionFence = { ...identity, controllerEpoch: 1, turnId: 'turn-one' }; + +class FakeGoalAdapter implements GoalSessionAdapter { + async publishOperationBarrier(): Promise {} + readonly provider = 'fake'; + readonly capabilities = { + nativeSessionId: 'eager', + steering: 'active_turn', + pause: 'active_turn', + modelChange: 'next_safe_boundary', + } as const; + openCalls = 0; + beginCalls = 0; + messageCalls: string[] = []; + pauseCalls = 0; + resumeCalls = 0; + resumeTurnCalls = 0; + resumeTurnAttempts: string[] = []; + modelCalls: string[] = []; + rejectedModel: string | undefined; + cancelCalls = 0; + events: GoalSessionEvent[] = []; + resumeEvents: GoalSessionEvent[] = []; + reconcileResult: GoalProviderReconcileResult = { outcome: 'failed', reason: 'not configured' }; + reconcileCalls = 0; + reconcileRequests: GoalProviderReconcileRequest[] = []; + openedWith: Array = []; + openAttempts: string[] = []; + turnStarted: (() => void) | undefined; + holdTurn: Promise | undefined; + + async openSession(request: GoalProviderOpenRequest): Promise { + this.openCalls += 1; + this.openAttempts.push(request.attemptId); + this.openedWith.push(request.persisted); + return request.persisted ?? { + providerSessionId: 'provider-session-stable', + recoveryMetadata: { checkpoint: 'created' }, + model: 'model-a', + }; + } + + async *beginTurn(_request: GoalBeginTurnRequest): AsyncIterable { + this.beginCalls += 1; + this.turnStarted?.(); + if (this.holdTurn) await this.holdTurn; + for (const event of this.events) yield event; + } + + async deliverMessage(request: GoalSteeringRequest): Promise<{ messageId: string }> { + this.messageCalls.push(request.messageId); + return { messageId: request.messageId }; + } + + async requestPause(_request: GoalPauseRequest): Promise<{ appliesAt: 'next_safe_boundary' }> { + this.pauseCalls += 1; + return { appliesAt: 'next_safe_boundary' }; + } + + async resumeSession(_request: GoalSessionControlFence, snapshot: GoalProviderSessionSnapshot): Promise { + this.resumeCalls += 1; + return snapshot; + } + + async *resumeTurn(request: GoalSessionFence & { executionId: string; attemptId: string }): AsyncIterable { + this.resumeTurnCalls += 1; + this.resumeTurnAttempts.push(request.attemptId); + for (const event of this.resumeEvents) yield event; + } + + async requestModelChange(request: GoalModelChangeRequest): Promise<{ requestedModel: string; appliesAt: 'immediate'; effectiveModel: string }> { + this.modelCalls.push(request.model); + if (request.model === this.rejectedModel) { + throw new UnsupportedGoalSessionTransitionError( + `Model transition to ${request.model} is unsupported by fake provider`, + 'UNSUPPORTED_MODEL_TRANSITION', + ); + } + return { requestedModel: request.model, appliesAt: 'immediate', effectiveModel: request.model }; + } + + async cancel(_request: GoalCancelRequest): Promise { + this.cancelCalls += 1; + } + + async reconcile(_request: GoalProviderReconcileRequest): Promise { + this.reconcileCalls += 1; + this.reconcileRequests.push(structuredClone(_request)); + return this.reconcileResult; + } +} + +async function openedRuntime(adapter = new FakeGoalAdapter()) { + const persistence = new InMemoryGoalSessionPorts(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const state = await supervisor.openSession({ ...identity, provider: 'fake', controllerEpoch: 1 }); + return { adapter, persistence, supervisor, state }; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +test('starts a recoverable turn and replays ordered normalized output and usage', async () => { + const adapter = new FakeGoalAdapter(); + adapter.events = [ + { type: 'output', channel: 'stdout', data: 'out\n' }, + { type: 'output', channel: 'stderr', data: 'warning\n' }, + { type: 'assistant', messageId: 'assistant-1', content: 'working' }, + { type: 'tool', toolCallId: 'tool-1', name: 'edit', phase: 'completed', data: { file: 'a.ts' } }, + { type: 'todo', todoId: 'todo-1', title: 'test', status: 'completed' }, + { type: 'usage', occurrenceId: 'usage-1', semantics: 'delta', watermark: 0, model: 'model-a', inputTokens: 12, outputTokens: 5 }, + { type: 'checkpoint', checkpointId: 'cp-1', recoveryMetadata: { checkpoint: 'cp-1' } }, + { type: 'completion', outcome: 'succeeded', summary: 'done' }, + ]; + const { persistence, supervisor } = await openedRuntime(adapter); + + const result = await supervisor.runTurn({ + ...fence, + executionId: 'execution-stable', + attemptId: 'attempt-unique', + objective: 'Implement the goal', + repository, + requestedModel: 'model-a', + }); + + assert.equal(result.disposition, 'started'); + assert.equal(adapter.beginCalls, 1); + assert.equal(result.state.status, 'idle'); + assert.deepEqual(result.state.recoveryMetadata, { checkpoint: 'cp-1' }); + const replay = await persistence.replay(identity); + assert.deepEqual(replay.map(value => value.sequence), [1, 2, 3, 4, 5, 6, 7, 8]); + assert.deepEqual(replay.slice(0, 2).map(value => value.event.type === 'output' ? value.event.channel : ''), ['stdout', 'stderr']); + assert.ok(replay.every(value => value.executionId === 'execution-stable' && value.attemptId === 'attempt-unique')); + assert.equal(replay[5].event.type, 'usage'); + const lateOutput = await persistence.append(fence, result.execution, { + type: 'output', channel: 'stdout', data: 'too late', + }); + assert.deepEqual(lateOutput, { accepted: false, reason: 'turn_not_active' }); +}); + +test('opens a new controller epoch by resuming the same persisted provider session', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence } = await openedRuntime(adapter); + const restartedSupervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'open-recovery-attempt'); + + const resumed = await restartedSupervisor.openSession({ ...identity, provider: 'fake', controllerEpoch: 2 }); + + assert.equal(resumed.controllerEpoch, 2); + assert.equal(resumed.providerSessionId, 'provider-session-stable'); + assert.equal(adapter.openCalls, 2); + assert.equal(adapter.openedWith[1]?.providerSessionId, 'provider-session-stable'); + assert.deepEqual(adapter.openedWith[1]?.recoveryMetadata, { checkpoint: 'created' }); + assert.equal(adapter.openAttempts[1], 'open-recovery-attempt'); + assert.notEqual(adapter.openAttempts[1], adapter.openAttempts[0]); +}); + +test('duplicate queue delivery cannot invoke a second provider turn', async () => { + const adapter = new FakeGoalAdapter(); + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + const { supervisor } = await openedRuntime(adapter); + const request = { + ...fence, + executionId: 'execution-stable', + objective: 'Only once', + repository, + requestedModel: 'model-a', + }; + + const first = supervisor.runTurn(request); + await started; + const duplicate = await supervisor.runTurn(request); + assert.equal(duplicate.disposition, 'duplicate'); + assert.equal(adapter.beginCalls, 1); + releaseTurn(); + await first; + const redeliveredAfterCompletion = await supervisor.runTurn(request); + assert.equal(redeliveredAfterCompletion.disposition, 'duplicate'); + assert.equal(adapter.beginCalls, 1); +}); + +test('a stale supervisor cannot append after controller takeover', async () => { + const adapter = new FakeGoalAdapter(); + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + adapter.events = [{ type: 'output', channel: 'stdout', data: 'stale' }]; + const { persistence, supervisor } = await openedRuntime(adapter); + const run = supervisor.runTurn({ + ...fence, + executionId: 'execution-one', + attemptId: 'attempt-one', + objective: 'Old owner', + repository, + requestedModel: 'model-a', + }); + await started; + await supervisor.takeover(identity, 2); + releaseTurn(); + + await assert.rejects(run, StaleGoalSessionFenceError); + assert.deepEqual(await persistence.replay(identity), []); + const rejected = await persistence.append(fence, { executionId: 'execution-one', attemptId: 'attempt-one' }, { + type: 'output', channel: 'stderr', data: 'also stale', + }); + assert.deepEqual(rejected, { accepted: false, reason: 'stale_fence' }); +}); + +test('delivers durable steering in order and acknowledges each ID once', async () => { + const adapter = new FakeGoalAdapter(); + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + const { persistence, supervisor } = await openedRuntime(adapter); + const run = supervisor.runTurn({ ...fence, executionId: 'execution-one', objective: 'Steer me', repository, requestedModel: 'model-a' }); + await started; + persistence.enqueueMessage({ ...identity, messageId: 'message-one', body: 'first' }); + persistence.enqueueMessage({ ...identity, messageId: 'message-two', body: 'second' }); + + await assert.rejects( + supervisor.deliverMessage({ ...fence, messageId: 'message-two', body: 'ignored caller copy' }), + /out of order/, + ); + assert.equal((await supervisor.deliverMessage({ ...fence, messageId: 'message-one', body: '' })).outcome, 'acknowledged'); + assert.equal((await supervisor.deliverMessage({ ...fence, messageId: 'message-one', body: '' })).outcome, 'acknowledged'); + assert.equal((await supervisor.deliverMessage({ ...fence, messageId: 'message-two', body: '' })).outcome, 'acknowledged'); + assert.deepEqual(adapter.messageCalls, ['message-one', 'message-two']); + releaseTurn(); + await run; +}); + +test('reports pause boundary, model effectiveness, same-turn resume, and terminal cancel separately', async () => { + const adapter = new FakeGoalAdapter(); + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + adapter.events = [ + { type: 'pause_boundary', boundary: 'after_tool', checkpointId: 'cp-pause', providerEventId: 'pause-after-tool-1' }, + ]; + adapter.resumeEvents = [ + { type: 'assistant', messageId: 'assistant-continued', content: 'resumed work' }, + { type: 'completion', outcome: 'succeeded', summary: 'done after resume' }, + ]; + const { persistence, supervisor } = await openedRuntime(adapter); + const runningTurn = supervisor.runTurn({ ...fence, executionId: 'execution-one', objective: 'Pause', repository, requestedModel: 'model-a' }); + await started; + const pauseAck = await supervisor.requestPause({ ...fence, reason: 'operator wants a checkpoint' }); + assert.deepEqual(pauseAck, { appliesAt: 'next_safe_boundary' }); + releaseTurn(); + const turn = await runningTurn; + assert.equal(turn.state.status, 'paused'); + + const modelAck = await supervisor.requestModelChange({ ...fence, model: 'model-b' }); + assert.deepEqual(modelAck, { requestedModel: 'model-b', appliesAt: 'immediate', effectiveModel: 'model-b' }); + // Resume continues the exact active turn to a single later completion. + const resumed = await supervisor.resumeTurn(fence); + assert.equal(resumed.disposition, 'started'); + assert.equal(resumed.state.status, 'idle'); + assert.equal(resumed.state.providerSessionId, 'provider-session-stable'); + assert.equal(adapter.resumeTurnCalls, 1); + assert.equal(adapter.beginCalls, 1); + const cancelled = await supervisor.cancel({ ...fence, reason: 'operator requested termination' }); + assert.equal(cancelled.status, 'terminated'); + assert.equal(adapter.resumeCalls, 1); + assert.equal(adapter.pauseCalls, 1); + assert.equal(adapter.cancelCalls, 1); + const types = (await persistence.replay(identity)).map(value => value.event.type); + assert.ok(types.includes('pause_boundary')); + assert.ok(types.includes('model_change_acknowledged')); + assert.ok(types.includes('model_changed')); + assert.ok(types.includes('session_resumed')); + assert.ok(types.includes('turn_resumed')); + // Exactly one turn completion, then the terminal cancel completion. + assert.equal(types.filter(type => type === 'completion').length, 2); + assert.equal(types.at(-1), 'completion'); +}); + +test('an unsupported model transition fails without replacing the provider session', async () => { + const adapter = new FakeGoalAdapter(); + adapter.rejectedModel = 'model-unsupported'; + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + const { persistence, supervisor } = await openedRuntime(adapter); + const running = supervisor.runTurn({ ...fence, executionId: 'execution-one', objective: 'Model test', repository, requestedModel: 'model-a' }); + await started; + + await assert.rejects( + supervisor.requestModelChange({ ...fence, model: 'model-unsupported' }), + (error: unknown) => error instanceof GoalSessionContractError + && !(error instanceof UnsupportedGoalSessionTransitionError) + && error.code === 'PROVIDER_OPERATION_FAILED', + ); + const unchanged = await persistence.load(identity); + assert.equal(unchanged?.currentModel, 'model-a'); + assert.equal(unchanged?.providerSessionId, 'provider-session-stable'); + releaseTurn(); + await running; +}); + +test('reconciles a missing container from durable provider and worktree state', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'daemon restarted' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: 'goal-branch', + observedHeadSha: 'abc123', + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + dirty: true, + }); + adapter.reconcileResult = { + outcome: 'resumed', + snapshot: { + providerSessionId: 'provider-session-stable', + recoveryMetadata: { checkpoint: 'recovered' }, + model: 'model-a', + }, + reason: 'Provider resumed from checkpoint against the inspected worktree', + }; + + const result = await supervisor.reconcile(identity, 2, repository); + assert.equal(result.outcome, 'resumed'); + assert.equal(result.state.controllerEpoch, 2); + assert.deepEqual(result.state.recoveryMetadata, { checkpoint: 'recovered' }); + assert.equal((await persistence.replay(identity)).at(-1)?.event.type, 'reconciliation'); +}); + +test('reconciliation accepts legitimate HEAD advancement and reports the current checkpoint separately', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + const advancedCheckout = { + ...repository, + repository: 'https://github.com/integry/propr.git', + headSha: 'def456', + }; + persistence.setContainerInspection(identity, { status: 'missing', reason: 'worker restarted' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedRepository: advancedCheckout.repository, + observedBranch: repository.branch, + observedHeadSha: advancedCheckout.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(advancedCheckout), + }); + adapter.reconcileResult = { outcome: 'alive', reason: 'goal commit is still recoverable' }; + + const result = await supervisor.reconcile(identity, 2, repository); + + assert.equal(result.outcome, 'alive'); + assert.equal(adapter.reconcileRequests[0]?.repository.observedHeadSha, 'def456'); + assert.equal(fingerprintGoalWorktree(repository), fingerprintGoalWorktree(advancedCheckout)); +}); + +test('reconciliation rejects repository replacement at the same path even without an expected HEAD', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + const headlessRepository = { + repository: repository.repository, + worktreePath: repository.worktreePath, + branch: repository.branch, + }; + const replacement = { ...headlessRepository, repository: 'https://github.com/foreign/replacement.git' }; + persistence.setContainerInspection(identity, { status: 'missing', reason: 'worker restarted' }); + persistence.setRepositoryInspection(headlessRepository, { + ...headlessRepository, + exists: true, + observedRepository: replacement.repository, + observedBranch: replacement.branch, + observedHeadSha: 'replacement-head', + observedWorktreeFingerprint: fingerprintGoalWorktree(replacement), + }); + adapter.reconcileResult = { outcome: 'alive', reason: 'must not inspect a foreign checkout' }; + + const result = await supervisor.reconcile(identity, 2, headlessRepository); + + assert.equal(result.outcome, 'blocked'); + assert.match(result.reason, /fingerprint mismatch/); + assert.equal(adapter.reconcileCalls, 0); +}); + +test('blocks reconciliation when authoritative container metadata is unavailable', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + persistence.setContainerInspection(identity, { status: 'daemon_unavailable', reason: 'socket unavailable' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: 'goal-branch', + observedHeadSha: 'abc123', + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { + outcome: 'failed', + reason: 'Provider checkpoint is corrupt and cannot be resumed', + }; + + const result = await supervisor.reconcile(identity, 2, repository); + + assert.equal(result.outcome, 'blocked'); + assert.equal(result.state.status, 'idle'); + assert.equal(adapter.reconcileCalls, 0); +}); + +test('blocks reconciliation with an actionable result when the worktree does not match', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'daemon restarted' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: 'unexpected-branch', + observedHeadSha: 'zzz999', + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { outcome: 'resumed', snapshot: { + providerSessionId: 'provider-session-stable', recoveryMetadata: { checkpoint: 'recovered' }, + }, reason: 'should not be reached' }; + + const result = await supervisor.reconcile(identity, 2, repository); + + assert.equal(result.outcome, 'blocked'); + assert.match(result.reason, /branch mismatch/); + // No provider side effect ran, and the session was not marked failed/resumed. + assert.equal(result.state.status, 'idle'); + const last = (await persistence.replay(identity)).at(-1); + assert.equal(last?.event.type, 'reconciliation'); + assert.equal(last?.event.type === 'reconciliation' ? last.event.outcome : undefined, 'blocked'); +}); + +test('resumes the exact paused turn on a replacement supervisor and completes once', async () => { + const adapter = new FakeGoalAdapter(); + adapter.events = [ + { type: 'assistant', messageId: 'a1', content: 'step one' }, + { type: 'checkpoint', checkpointId: 'cp-1', recoveryMetadata: { checkpoint: 'cp-1' } }, + { type: 'pause_boundary', boundary: 'after_tool', checkpointId: 'cp-1', providerEventId: 'pause-after-tool-2' }, + ]; + adapter.resumeEvents = [ + { type: 'assistant', messageId: 'a2', content: 'step two' }, + { type: 'usage', occurrenceId: 'usage-resume-1', semantics: 'delta', watermark: 0, model: 'model-a', inputTokens: 3, outputTokens: 4 }, + { type: 'completion', outcome: 'succeeded', summary: 'finished after restart' }, + ]; + const { persistence, supervisor } = await openedRuntime(adapter); + const first = await supervisor.runTurn({ + ...fence, executionId: 'execution-one', attemptId: 'attempt-one', + objective: 'Long turn', repository, requestedModel: 'model-a', + }); + assert.equal(first.disposition, 'started'); + assert.equal(first.state.status, 'paused'); + + // Simulate a worker/container restart: a brand-new supervisor takes over. + const replacement = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'attempt-recovery'); + await replacement.takeover(identity, 2); + const resumeFence: GoalSessionControlFence = { ...identity, controllerEpoch: 2 }; + const resumed = await replacement.resumeTurn(resumeFence); + + assert.equal(resumed.disposition, 'started'); + assert.equal(resumed.state.status, 'idle'); + assert.equal(adapter.beginCalls, 1, 'the provider turn is invoked exactly once'); + assert.equal(adapter.resumeTurnCalls, 1); + assert.equal(resumed.execution.executionId, 'execution-one'); + assert.equal(resumed.execution.attemptId, 'attempt-recovery'); + + const replay = await persistence.replay(identity); + const types = replay.map(event => event.event.type); + assert.deepEqual(replay.map(event => event.sequence), replay.map((_, index) => index + 1)); + assert.equal(types.filter(type => type === 'completion').length, 1, 'the turn completes exactly once'); + const turnResumed = replay.find(event => event.event.type === 'turn_resumed'); + assert.equal(turnResumed?.event.type === 'turn_resumed' ? turnResumed.event.turnId : undefined, 'turn-one'); + assert.equal(types.at(-1), 'completion'); +}); + +class DeterministicAdapter extends FakeGoalAdapter { + readonly supportsDeterministicOpen = true; + lastOpenKey: string | undefined; + lastAttemptId: string | undefined; + + async openSession(request: GoalProviderOpenRequest): Promise { + this.openCalls += 1; + this.lastOpenKey = request.deterministicOpenKey; + this.lastAttemptId = request.attemptId; + this.openedWith.push(request.persisted); + return request.persisted ?? { + providerSessionId: 'provider-deterministic', + recoveryMetadata: { checkpoint: 'created' }, + model: 'model-a', + }; + } +} + +test('recovers a crash before provider-identity persistence when the provider is deterministic', async () => { + const persistence = new InMemoryGoalSessionPorts(); + const adapter = new DeterministicAdapter(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'attempt-fresh'); + const timestamp = new Date().toISOString(); + // A previous controller recorded initialization intent, then crashed before + // persisting the provider session identity. + await persistence.create({ + ...identity, provider: 'fake', controllerEpoch: 1, status: 'initializing', + completedTurnIds: [], + initializationIntent: { attemptId: 'attempt-x', deterministicOpenKey: 'key-x', recordedAt: timestamp }, + createdAt: timestamp, updatedAt: timestamp, + }); + + const recovered = await supervisor.openSession({ ...identity, provider: 'fake', controllerEpoch: 2 }); + + assert.equal(recovered.status, 'idle'); + assert.equal(recovered.providerSessionId, 'provider-deterministic'); + assert.equal(recovered.initializationIntent, undefined); + assert.equal(adapter.lastOpenKey, 'key-x'); + assert.equal(adapter.lastAttemptId, 'attempt-fresh'); + assert.notEqual(adapter.lastAttemptId, 'attempt-x'); +}); + +test('fails an unrecoverable crash before provider-identity persistence when open is not deterministic', async () => { + const persistence = new InMemoryGoalSessionPorts(); + const adapter = new FakeGoalAdapter(); + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const timestamp = new Date().toISOString(); + await persistence.create({ + ...identity, provider: 'fake', controllerEpoch: 1, status: 'initializing', + completedTurnIds: [], createdAt: timestamp, updatedAt: timestamp, + }); + + await assert.rejects( + supervisor.openSession({ ...identity, provider: 'fake', controllerEpoch: 2 }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'INCOMPLETE_INITIALIZATION', + ); +}); + +class ThrowingBeginAdapter extends FakeGoalAdapter { + beginTurn(): AsyncIterable { + this.beginCalls += 1; + throw new Error('begin invocation exploded'); + } +} + +class ThrowingResumeAdapter extends FakeGoalAdapter { + resumeTurn(): AsyncIterable { + this.resumeTurnCalls += 1; + throw new Error('resume invocation exploded'); + } +} + +test('a synchronous begin-turn invocation failure fences the session as failed with one completion', async () => { + const adapter = new ThrowingBeginAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + + await assert.rejects( + supervisor.runTurn({ ...fence, executionId: 'exec-b', attemptId: 'att-b', objective: 'boom', repository, requestedModel: 'model-a' }), + /Provider operation failed safely/, + ); + + const state = await persistence.load(identity); + assert.equal(state?.status, 'failed'); + assert.equal(state?.activeTurn?.status, 'failed'); + const completions = (await persistence.replay(identity)).filter(event => event.event.type === 'completion'); + assert.equal(completions.length, 1); + assert.equal(completions[0].event.type === 'completion' ? completions[0].event.outcome : '', 'failed'); +}); + +test('a synchronous resume-turn invocation failure fences the session as failed with one completion', async () => { + const adapter = new ThrowingResumeAdapter(); + let releaseTurn!: () => void; + adapter.holdTurn = new Promise(resolve => { releaseTurn = resolve; }); + const started = new Promise(resolve => { adapter.turnStarted = resolve; }); + adapter.events = [{ + type: 'pause_boundary', boundary: 'after_tool', checkpointId: 'cp-pause', providerEventId: 'pause-after-tool-3', + }]; + const { persistence, supervisor } = await openedRuntime(adapter); + const running = supervisor.runTurn({ ...fence, executionId: 'exec-r', attemptId: 'att-r', objective: 'pause then resume', repository, requestedModel: 'model-a' }); + await started; + await supervisor.requestPause({ ...fence, reason: 'checkpoint' }); + releaseTurn(); + const paused = await running; + assert.equal(paused.state.status, 'paused'); + + await assert.rejects(supervisor.resumeTurn(fence), /Provider operation failed safely/); + + const state = await persistence.load(identity); + assert.equal(state?.status, 'failed'); + assert.equal(state?.activeTurn?.status, 'failed'); + const completions = (await persistence.replay(identity)).filter(event => event.event.type === 'completion'); + assert.equal(completions.length, 1); +}); + +test('rejects a model change on a terminated session before calling the adapter', async () => { + const adapter = new FakeGoalAdapter(); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + const { supervisor } = await openedRuntime(adapter); + await supervisor.runTurn({ ...fence, executionId: 'exec-one', objective: 'run', repository, requestedModel: 'model-a' }); + await supervisor.cancel({ ...fence, reason: 'operator requested termination' }); + const callsBefore = adapter.modelCalls.length; + + await assert.rejects( + supervisor.requestModelChange({ ...fence, model: 'model-b' }), + (error: unknown) => error instanceof GoalSessionContractError && error.code === 'SESSION_NOT_CONTROLLABLE', + ); + assert.equal(adapter.modelCalls.length, callsBefore, 'the adapter must not be called for a terminated session'); +}); + +test('reconciles a running turn after container loss into a resumable turn a replacement continues once', async () => { + const persistence = new InMemoryGoalSessionPorts(); + const adapter = new FakeGoalAdapter(); + adapter.resumeEvents = [ + { type: 'assistant', messageId: 'a2', content: 'continued after recovery' }, + { type: 'completion', outcome: 'succeeded', summary: 'finished after container loss' }, + ]; + const timestamp = new Date().toISOString(); + await persistence.create({ + ...identity, provider: 'fake', controllerEpoch: 1, status: 'running', + providerSessionId: 'provider-session-stable', + recoveryMetadata: { checkpoint: 'mid-turn' }, + currentModel: 'model-a', requestedModel: 'model-a', + activeTurn: { + executionId: 'execution-live', attemptId: 'attempt-live', turnId: 'turn-one', executionEpoch: 1, + objective: 'long turn', requestedModel: 'model-a', repository, status: 'running', + }, + completedTurnIds: [], createdAt: timestamp, updatedAt: timestamp, + }); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'container lost' }); + persistence.setRepositoryInspection(repository, { + ...repository, exists: true, observedBranch: 'goal-branch', observedHeadSha: 'abc123', + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { + outcome: 'resumed', + snapshot: { providerSessionId: 'provider-session-stable', recoveryMetadata: { checkpoint: 'recovered' }, model: 'model-a' }, + reason: 'provider resumed from checkpoint', + }; + + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const result = await supervisor.reconcile(identity, 2, repository); + assert.equal(result.outcome, 'resumed'); + // The interrupted running turn is reconciled into an explicitly resumable + // paused turn, never left idle where a new turn could overwrite it. + assert.equal(result.state.status, 'paused'); + assert.equal(result.state.activeTurn?.status, 'paused'); + assert.equal(result.state.activeTurn?.executionId, 'execution-live'); + assert.notEqual(result.state.activeTurn?.attemptId, 'attempt-live'); + assert.equal(result.state.activeTurn?.attemptId, adapter.reconcileRequests[0].attemptId); + + const replacement = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'attempt-recovered'); + const resumed = await replacement.resumeTurn({ ...identity, controllerEpoch: 2 }); + assert.equal(resumed.disposition, 'started'); + assert.equal(resumed.state.status, 'idle'); + assert.equal(resumed.execution.executionId, 'execution-live'); + assert.equal(resumed.execution.attemptId, 'attempt-recovered'); + assert.notEqual(resumed.execution.attemptId, adapter.reconcileRequests[0].attemptId); + assert.equal(adapter.resumeTurnCalls, 1); + assert.equal(adapter.beginCalls, 0, 'no new turn was begun for the recovered execution'); + const completions = (await persistence.replay(identity)).filter(event => event.event.type === 'completion'); + assert.equal(completions.length, 1, 'the recovered turn completes exactly once'); +}); + +test('blocks reconciliation when the worktree branch cannot actually be observed', async () => { + const adapter = new FakeGoalAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'daemon restarted' }); + // The worktree exists but its git state could not be inspected. + persistence.setRepositoryInspection(repository, { + ...repository, exists: true, reason: 'git rev-parse failed: not a git repository', + }); + adapter.reconcileResult = { + outcome: 'resumed', + snapshot: { providerSessionId: 'provider-session-stable', recoveryMetadata: { checkpoint: 'x' } }, + reason: 'should not be reached', + }; + + const result = await supervisor.reconcile(identity, 2, repository); + assert.equal(result.outcome, 'blocked'); + assert.match(result.reason, /branch could not be observed/); + assert.equal(result.state.status, 'idle'); +}); + +test('redelivery of an older completed turn recovers its original execution identity', async () => { + const adapter = new FakeGoalAdapter(); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + const { supervisor } = await openedRuntime(adapter); + const firstReq = { + ...fence, turnId: 'turn-one', executionId: 'exec-one', attemptId: 'attempt-one', + objective: 'first', repository, requestedModel: 'model-a', + }; + await supervisor.runTurn(firstReq); + // A subsequent turn replaces activeTurn with a different execution identity. + await supervisor.runTurn({ + ...fence, turnId: 'turn-two', executionId: 'exec-two', attemptId: 'attempt-two', + objective: 'second', repository, requestedModel: 'model-a', + }); + assert.equal(adapter.beginCalls, 2); + + const redelivered = await supervisor.runTurn(firstReq); + assert.equal(redelivered.disposition, 'duplicate'); + assert.equal(redelivered.disposition === 'duplicate' && redelivered.reattached, true); + assert.equal(redelivered.execution.executionId, 'exec-one'); + assert.equal(redelivered.execution.attemptId, 'attempt-one'); + assert.equal(adapter.beginCalls, 2, 'the older turn is not re-invoked on the provider'); +}); + +test('goal-scoped session state cannot be read or reused by another goal', async () => { + const { persistence } = await openedRuntime(); + await assert.rejects( + persistence.load({ goalId: 'different-goal', sessionId: identity.sessionId }), + GoalSessionScopeError, + ); +}); + +test('a delayed same-epoch model acknowledgement cannot overwrite a newer intent', async () => { + const gate = deferred(); + const started = deferred(); + class RacingModelAdapter extends FakeGoalAdapter { + override async requestModelChange(request: GoalModelChangeRequest) { + this.modelCalls.push(request.model); + if (request.model === 'model-old') { + started.resolve(); + await gate.promise; + } + return { requestedModel: request.model, appliesAt: 'immediate' as const, effectiveModel: request.model }; + } + } + const adapter = new RacingModelAdapter(); + const { persistence, supervisor } = await openedRuntime(adapter); + const oldRequest = supervisor.requestModelChange({ ...fence, model: 'model-old' }); + await started.promise; + await supervisor.requestModelChange({ ...fence, model: 'model-new' }); + gate.resolve(); + await assert.rejects(oldRequest, StaleGoalSessionFenceError); + + const state = await persistence.load(identity); + assert.equal(state?.requestedModel, 'model-new'); + assert.equal(state?.currentModel, 'model-new'); + const changedModels = (await persistence.replay(identity)).flatMap(record => + record.event.type === 'model_changed' ? [record.event.model] : []); + assert.deepEqual(changedModels, ['model-new']); +}); + +test('each failed recovery retry durably advances to another fresh attempt', async () => { + class RetryResumeAdapter extends FakeGoalAdapter { + failResume = true; + override async resumeSession(request: GoalSessionControlFence, snapshot: GoalProviderSessionSnapshot) { + if (this.failResume) { + this.failResume = false; + throw new Error('recovery transport failed'); + } + return super.resumeSession(request, snapshot); + } + } + const adapter = new RetryResumeAdapter(); + adapter.events = [{ type: 'pause_boundary', boundary: 'checkpoint', providerEventId: 'pause-checkpoint-1' }]; + adapter.resumeEvents = [{ type: 'completion', outcome: 'succeeded' }]; + const persistence = new InMemoryGoalSessionPorts(); + const ids = ['provider-open-attempt', 'attempt-recovery-one', 'attempt-recovery-two']; + const supervisor = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => ids.shift()!); + await supervisor.openSession({ ...identity, provider: 'fake', controllerEpoch: 1 }); + await supervisor.runTurn({ + ...fence, executionId: 'execution-retry', attemptId: 'attempt-crashed', + objective: 'retry recovery', repository, requestedModel: 'model-a', + }); + + await assert.rejects(supervisor.resumeTurn(fence), /Provider operation failed safely/); + assert.equal((await persistence.load(identity))?.activeTurn?.attemptId, 'attempt-recovery-one'); + assert.equal((await persistence.load(identity))?.status, 'paused'); + assert.deepEqual(await persistence.append(fence, { + executionId: 'execution-retry', attemptId: 'attempt-crashed', + }, { type: 'output', channel: 'stdout', data: 'late crashed output' }), { + accepted: false, reason: 'turn_not_active', + }); + const recovered = await supervisor.resumeTurn(fence); + assert.equal(recovered.execution.attemptId, 'attempt-recovery-two'); + assert.deepEqual(adapter.resumeTurnAttempts, ['attempt-recovery-two']); +}); + +test('a delayed same-epoch resume cannot resurrect a terminal session', async () => { + const gate = deferred(); + const started = deferred(); + class RacingResumeAdapter extends FakeGoalAdapter { + override async resumeSession(_request: GoalSessionControlFence, snapshot: GoalProviderSessionSnapshot) { + started.resolve(); + await gate.promise; + return snapshot; + } + } + const adapter = new RacingResumeAdapter(); + adapter.events = [{ type: 'pause_boundary', boundary: 'checkpoint', providerEventId: 'pause-checkpoint-2' }]; + const { persistence, supervisor } = await openedRuntime(adapter); + await supervisor.runTurn({ + ...fence, executionId: 'execution-race', attemptId: 'attempt-crashed', + objective: 'pause', repository, requestedModel: 'model-a', + }); + const resume = supervisor.resumeTurn(fence); + await started.promise; + await supervisor.cancel({ ...fence, reason: 'terminal wins' }); + gate.resolve(); + await assert.rejects(resume, StaleGoalSessionFenceError); + + const state = await persistence.load(identity); + assert.equal(state?.status, 'terminated'); + assert.equal(state?.activeTurn, undefined); + const completions = (await persistence.replay(identity)).filter(record => record.event.type === 'completion'); + assert.equal(completions.length, 1); + assert.equal(completions[0].event.type === 'completion' ? completions[0].event.outcome : '', 'cancelled'); +}); + +test('terminal transaction survives an ambiguous post-commit crash without duplicate completion', async () => { + const adapter = new FakeGoalAdapter(); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + const { persistence, supervisor } = await openedRuntime(adapter); + const request = { + ...fence, executionId: 'execution-atomic', attemptId: 'attempt-atomic', + objective: 'atomic completion', repository, requestedModel: 'model-a', + }; + persistence.setTerminalFault('after_commit'); + await assert.rejects(supervisor.runTurn(request), /Injected crash after terminal transaction commit/); + assert.equal((await persistence.load(identity))?.status, 'idle'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); + + const restarted = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()); + const duplicate = await restarted.runTurn(request); + assert.equal(duplicate.disposition, 'duplicate'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('a pre-commit crash leaves neither terminal state nor event and recovery completes atomically', async () => { + const adapter = new FakeGoalAdapter(); + adapter.events = [{ type: 'completion', outcome: 'succeeded' }]; + adapter.resumeEvents = [{ type: 'completion', outcome: 'succeeded' }]; + const { persistence, supervisor } = await openedRuntime(adapter); + persistence.setTerminalFault('before_commit_always'); + await assert.rejects(supervisor.runTurn({ + ...fence, executionId: 'execution-window', attemptId: 'attempt-crashed', + objective: 'crash window', repository, requestedModel: 'model-a', + }), /Injected crash before terminal transaction commit/); + assert.equal((await persistence.load(identity))?.status, 'running'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 0); + + persistence.setTerminalFault(undefined); + persistence.setContainerInspection(identity, { status: 'missing', reason: 'worker crashed' }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { + outcome: 'resumed', + reason: 'checkpoint recovered', + snapshot: { providerSessionId: 'provider-session-stable', recoveryMetadata: { checkpoint: 'recovered' } }, + }; + await supervisor.reconcile(identity, 2, repository); + const restarted = new GoalSessionSupervisor(adapter, persistence.asRuntimePorts(), () => 'attempt-recovered'); + const recovered = await restarted.resumeTurn({ ...identity, controllerEpoch: 2 }); + assert.equal(recovered.execution.attemptId, 'attempt-recovered'); + assert.equal((await persistence.load(identity))?.status, 'idle'); + assert.equal((await persistence.replay(identity)).filter(record => record.event.type === 'completion').length, 1); +}); + +test('reconciliation requires every authoritative container identity field', async () => { + const expectedIdentity = { + ...identity, + executionEpoch: 1, + turnId: fence.turnId, + attemptId: 'attempt-live', + worktreeFingerprint: fingerprintGoalWorktree(repository), + }; + const variants: Array<[string, Partial | null, boolean]> = [ + ['exact identity', {}, true], + ['missing metadata', null, false], + ['foreign goal', { goalId: 'other-goal' }, false], + ['stale epoch', { executionEpoch: 0 }, false], + ['foreign turn', { turnId: 'other-turn' }, false], + ['stale attempt', { attemptId: 'old-attempt' }, false], + ['foreign worktree', { worktreeFingerprint: 'wrong-fingerprint' }, false], + ]; + for (const [label, replacement, accepted] of variants) { + const persistence = new InMemoryGoalSessionPorts(); + const adapter = new FakeGoalAdapter(); + const timestamp = new Date().toISOString(); + await persistence.create({ + ...identity, + provider: 'fake', controllerEpoch: 1, status: 'running', + providerSessionId: 'provider-session-stable', recoveryMetadata: { checkpoint: 'live' }, + activeTurn: { + executionId: 'execution-live', attemptId: 'attempt-live', executionEpoch: 1, + turnId: fence.turnId, objective: 'live', requestedModel: 'model-a', repository, status: 'running', + }, + completedTurnIds: [], createdAt: timestamp, updatedAt: timestamp, + }); + persistence.setContainerInspection(identity, { + status: 'running', + recoveryIdentity: replacement ? { ...expectedIdentity, ...replacement } : undefined, + }); + persistence.setRepositoryInspection(repository, { + ...repository, + exists: true, + observedBranch: repository.branch, + observedHeadSha: repository.headSha, + observedWorktreeFingerprint: fingerprintGoalWorktree(repository), + }); + adapter.reconcileResult = { outcome: 'alive', reason: 'identity accepted' }; + const result = await new GoalSessionSupervisor(adapter, persistence.asRuntimePorts()) + .reconcile(identity, 2, repository); + assert.equal(result.outcome, accepted ? 'alive' : 'blocked', label); + assert.equal(adapter.reconcileCalls, accepted ? 1 : 0, label); + } +}); diff --git a/packages/core/test/productionGoalSessionTestSupport.ts b/packages/core/test/productionGoalSessionTestSupport.ts new file mode 100644 index 000000000..2556cbf9a --- /dev/null +++ b/packages/core/test/productionGoalSessionTestSupport.ts @@ -0,0 +1,40 @@ +import Database from 'better-sqlite3'; +import knex from 'knex'; +import type { GoalSessionRecoveryPort } from '../src/agents/goalSession/runtimePorts.js'; + +export const recovery: GoalSessionRecoveryPort = { + inspectContainer: async () => ({ status: 'missing', reason: 'test' }), + inspectRepository: async repository => ({ ...repository, exists: true }), +}; + +/** Runs the real #2018 foundation/replay migrations and the runtime leaf. */ +export async function createProductionSchema(filename: string): Promise { + const client = knex({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true }); + try { + const foundation = await import('../src/db/migrations/20260831000000_create_goal_control_plane.js'); + const replay = await import('../src/db/migrations/20260901000000_add_durable_goal_replay.js'); + const runtime = await import('../src/db/migrations/20260902000000_extend_goal_control_provider_effects.js'); + await foundation.up(client); + await replay.up(client); + await runtime.up(client); + } finally { + await client.destroy(); + } +} + +export function seedAuthoritativeGoal( + database: Database.Database, + options: { goalId: string; agent: string; model?: string; leaseEpoch?: number }, +): void { + const now = new Date().toISOString(); + database.prepare(`INSERT INTO goals + (goal_id, owner_user_id, repository, objective, state, agent, requested_model, + effective_model, lease_owner, lease_epoch, lease_expires_at, created_at, updated_at) + VALUES (?, 'test-owner', 'integry/propr', 'runtime composition test', 'running', ?, ?, ?, + 'runtime-controller', ?, ?, ?, ?)`) + .run(options.goalId, options.agent, options.model ?? 'model-a', options.model ?? 'model-a', + options.leaseEpoch ?? 1, new Date(Date.now() + 60_000).toISOString(), now, now); + database.prepare(`INSERT OR IGNORE INTO goal_event_state + (goal_id, high_watermark, min_retained_sequence, projection_sequence, checkpoint_sequence, updated_at) + VALUES (?, 0, 1, 0, 0, ?)`).run(options.goalId, now); +} diff --git a/packages/core/test/supervisedDockerBackpressure.test.ts b/packages/core/test/supervisedDockerBackpressure.test.ts new file mode 100644 index 000000000..bb8acbd70 --- /dev/null +++ b/packages/core/test/supervisedDockerBackpressure.test.ts @@ -0,0 +1,131 @@ +import assert from 'node:assert/strict'; +import * as actualChildProcess from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { mock, test } from 'node:test'; + +function pausableStream() { + return Object.assign(new EventEmitter(), { + paused: false, + pause(): void { this.paused = true; }, + resume(): void { this.paused = false; }, + }); +} + +const child = Object.assign(new EventEmitter(), { + stdout: pausableStream(), + stderr: pausableStream(), + stdin: { destroyed: false, writableEnded: false, write(_d: string, cb: (e?: Error | null) => void) { cb(); return true; }, end() { this.writableEnded = true; } }, + exitCode: null as number | null, + kill: mock.fn(() => true), +}); + +await mock.module('child_process', { + namedExports: { + ...actualChildProcess, + spawn: mock.fn(() => child), + execFileSync: mock.fn(), + }, +}); + +const { executeSupervisedDockerCommand } = await import('../src/claude/docker/supervisedDockerExecutor.js'); + +const tick = (): Promise => new Promise(resolve => setImmediate(resolve)); +const recoveryIdentity = { + executionId: 'execution', attemptId: 'attempt', worktreeFingerprint: 'worktree', + operationGeneration: 1, operationKind: 'turn' as const, operationId: 'turn-effect', +}; + +test('a slow sink pauses the source streams and preserves ordering without unbounded buffering', async () => { + const received: string[] = []; + const gates: Array<() => void> = []; + const execution = executeSupervisedDockerCommand(['run', 'img'], { + goalId: 'g', sessionId: 's', controllerEpoch: 1, turnId: 't', ...recoveryIdentity, + maxQueuedBytes: 200, + durableOutput: output => { + received.push(output.data); + return new Promise(resolve => { gates.push(resolve); }); + }, + }); + + const chunk = 'x'.repeat(30); + for (let i = 0; i < 4; i += 1) child.stdout.emit('data', Buffer.from(`${i}${chunk}`)); + await tick(); + + // The first chunk is in flight; the rest are queued past the high-water mark, + // so the source stream is paused and memory stays bounded. + assert.equal(child.stdout.paused, true); + assert.equal(received.length, 1); + + // Release deliveries one at a time; ordering is preserved and the stream + // resumes once the backlog drains below the low-water mark. + while (gates.length) { + const release = gates.shift()!; + release(); + await tick(); + } + assert.equal(child.stdout.paused, false); + assert.deepEqual(received.map(value => value[0]), ['0', '1', '2', '3']); + + child.exitCode = 0; + child.emit('close', 0); + assert.deepEqual(await execution.completion, { exitCode: 0 }); +}); + +test('exceeding the queued-byte bound cancels with an actionable overflow error', async () => { + const execution = executeSupervisedDockerCommand(['run', 'img'], { + goalId: 'g', sessionId: 's', controllerEpoch: 1, turnId: 't2', ...recoveryIdentity, + maxQueuedBytes: 16, + // Never resolves: simulates a sink that is permanently too slow. + durableOutput: () => new Promise(() => {}), + }); + + child.stdout.emit('data', Buffer.from('y'.repeat(64))); + await tick(); + child.emit('close', 0); + + await assert.rejects(execution.completion, /backpressure bound/); +}); + +test('rejects non-positive, non-finite, or incoherent backpressure limits', () => { + const base = { goalId: 'g', sessionId: 's', controllerEpoch: 1, turnId: 'limits', ...recoveryIdentity, durableOutput: () => {} }; + assert.throws(() => executeSupervisedDockerCommand(['run', 'img'], { ...base, maxChunkBytes: 0 }), /maxChunkBytes must be a positive safe integer/); + assert.throws(() => executeSupervisedDockerCommand(['run', 'img'], { ...base, maxChunkBytes: -8 }), /maxChunkBytes must be a positive safe integer/); + assert.throws(() => executeSupervisedDockerCommand(['run', 'img'], { ...base, maxQueuedBytes: Number.POSITIVE_INFINITY }), /maxQueuedBytes must be a positive safe integer/); + assert.throws(() => executeSupervisedDockerCommand(['run', 'img'], { ...base, maxQueuedBytes: 1.5 }), /maxQueuedBytes must be a positive safe integer/); + assert.throws(() => executeSupervisedDockerCommand(['run', 'img'], { ...base, maxChunkBytes: 128, maxQueuedBytes: 64 }), /must not exceed maxQueuedBytes/); +}); + +test('a single oversized read is stopped during enqueue by the hard cap', async () => { + const execution = executeSupervisedDockerCommand(['run', 'img'], { + goalId: 'g', sessionId: 's', controllerEpoch: 1, turnId: 'oversized', ...recoveryIdentity, + maxChunkBytes: 16, maxQueuedBytes: 64, + // Never drains, so the whole read can only be bounded by enqueue-time enforcement. + durableOutput: () => new Promise(() => {}), + }); + + child.stdout.emit('data', Buffer.from('z'.repeat(4096))); + await tick(); + child.emit('close', 0); + + await assert.rejects(execution.completion, /backpressure bound/); +}); + +test('splitting a large read preserves multi-byte UTF-8 characters across chunk boundaries', async () => { + const received: string[] = []; + const execution = executeSupervisedDockerCommand(['run', 'img'], { + goalId: 'g', sessionId: 's', controllerEpoch: 1, turnId: 'utf8', ...recoveryIdentity, + maxChunkBytes: 4, maxQueuedBytes: 1_000_000, + durableOutput: output => { received.push(output.data); }, + }); + + const text = '你好世界🌍émojî'.repeat(8); + child.stdout.emit('data', Buffer.from(text, 'utf8')); + await tick(); + child.exitCode = 0; + child.emit('close', 0); + await execution.completion; + + assert.ok(received.length > 1, 'the read was split into multiple durable chunks'); + assert.equal(received.join(''), text); + assert.ok(!received.join('').includes('�'), 'no UTF-8 replacement characters were produced'); +}); diff --git a/packages/core/test/supervisedDockerExecutor.test.ts b/packages/core/test/supervisedDockerExecutor.test.ts new file mode 100644 index 000000000..9e58a7434 --- /dev/null +++ b/packages/core/test/supervisedDockerExecutor.test.ts @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict'; +import * as actualChildProcess from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { mock, test } from 'node:test'; + +const spawnCalls: Array<{ command: string; args: string[] }> = []; +const writtenInput: string[] = []; +const child = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + stdin: { + destroyed: false, + writableEnded: false, + write(data: string, callback: (error?: Error | null) => void) { + writtenInput.push(data); + callback(); + return true; + }, + end() { this.writableEnded = true; }, + }, + exitCode: null as number | null, + signalCode: null as NodeJS.Signals | null, + kill: mock.fn(() => true), +}); + +await mock.module('child_process', { + namedExports: { + ...actualChildProcess, + spawn: mock.fn((command: string, args: string[]) => { + spawnCalls.push({ command, args }); + return child; + }), + execFileSync: mock.fn(), + }, +}); + +const { executeSupervisedDockerCommand } = await import('../src/claude/docker/dockerExecutor.js'); + +test('duplex Docker execution fences labels, keeps stdin open, and durably orders stream chunks', async () => { + const output: Array<{ channel: string; data: string }> = []; + const execution = executeSupervisedDockerCommand( + ['run', '--name', 'goal-container', 'propr/agent:test', 'agent-command'], + { + goalId: 'goal-one', + sessionId: 'session-one', + controllerEpoch: 7, + turnId: 'turn-one', + executionId: 'execution-one', + attemptId: 'attempt-one', + worktreeFingerprint: 'worktree-one', + operationGeneration: 8, + operationKind: 'turn', + operationId: 'turn-one-effect', + durableOutput: async event => { + await Promise.resolve(); + output.push({ channel: event.channel, data: event.data }); + }, + }, + ); + + await execution.writeInput('corrective message\n'); + assert.equal(child.stdin.writableEnded, false); + child.stdout.emit('data', Buffer.from('stdout-one')); + child.stderr.emit('data', Buffer.from('stderr-one')); + child.exitCode = 0; + child.emit('close', 0); + + assert.deepEqual(await execution.completion, { exitCode: 0 }); + assert.deepEqual(writtenInput, ['corrective message\n']); + assert.deepEqual(output, [ + { channel: 'stdout', data: 'stdout-one' }, + { channel: 'stderr', data: 'stderr-one' }, + ]); + const args = spawnCalls[0].args; + assert.ok(args.includes('propr.goal.id=goal-one')); + assert.ok(args.includes('propr.goal.session=session-one')); + assert.ok(args.includes('propr.goal.controller-epoch=7')); + assert.ok(args.includes('propr.goal.turn=turn-one')); + assert.ok(args.includes('propr.goal.execution=execution-one')); + assert.ok(args.includes('propr.goal.attempt=attempt-one')); + assert.ok(args.includes('propr.goal.worktree-fingerprint=worktree-one')); + assert.equal(execution.containerName, 'goal-container'); +}); + +test('raw durable output is a secret-free allowlist even with poisoned runtime excess properties', async () => { + const delivered: unknown[] = []; + const secretValues = [ + 'runtime-api-token', + '/host/credential/source.json', + '/host/private/worktree', + 'provider --token hidden-command-secret', + 'arbitrary-extra-secret', + ]; + const options = { + goalId: 'goal-safe-output', + sessionId: 'session-safe-output', + controllerEpoch: 9, + turnId: 'turn-safe-output', + executionId: 'execution-safe-output', + attemptId: 'attempt-safe-output', + worktreeFingerprint: 'public-worktree-fingerprint', + operationGeneration: 10, + operationKind: 'turn' as const, + operationId: 'turn-safe-output-effect', + env: { API_TOKEN: secretValues[0] }, + cwd: secretValues[2], + request: { environment: { API_TOKEN: secretValues[0] }, command: secretValues[3] }, + credentialMounts: [{ source: secretValues[1], target: '/container/credential' }], + arbitraryExtra: secretValues[4], + durableOutput: (output: unknown) => { delivered.push(structuredClone(output)); }, + }; + const execution = executeSupervisedDockerCommand(['run', 'img'], options); + child.stdout.emit('data', Buffer.from('public output')); + child.emit('close', 0); + await execution.completion; + + assert.equal(delivered.length, 1); + assert.deepEqual(Object.keys(delivered[0] as object).sort(), [ + 'attemptId', 'channel', 'controllerEpoch', 'data', 'executionId', 'goalId', + 'operationGeneration', 'operationId', 'operationKind', 'operationLeaseExpiresAt', 'recordedAt', 'sequence', + 'sessionId', 'turnId', 'worktreeFingerprint', + ]); + const raw = JSON.stringify(delivered[0]); + for (const secret of secretValues) assert.ok(!raw.includes(secret), `raw delivery leaked ${secret}`); +});