diff --git a/packages/cli/src/extensions/goal/evaluator.ts b/packages/cli/src/extensions/goal/evaluator.ts index ca9322cb0..bb00e04d0 100644 --- a/packages/cli/src/extensions/goal/evaluator.ts +++ b/packages/cli/src/extensions/goal/evaluator.ts @@ -5,7 +5,7 @@ * * - `keywordGoalEvaluator`: fast, local keyword check. * - `createLlmGoalEvaluator(...)`: sends a compact transcript + goal to a - * small, fast model (default Anthropic Haiku) and parses a yes/no decision. + * model (the session's, or a pinned one) and parses a yes/no decision. */ import type { AssistantMessage, @@ -199,6 +199,12 @@ export interface LlmEvaluatorDeps { export interface SessionCacheEvaluatorDeps extends LlmEvaluatorDeps { /** The session prefix to reuse, exactly as pi last sent it. */ captured: CapturedSessionContext; + /** + * The id pi sends as `sessionId` on the session's own calls. Providers key + * their prompt cache on it (openai-codex `prompt_cache_key`); without it + * the judge call always misses and the evaluator downgrades for good. + */ + sessionId?: string; } /** @@ -270,6 +276,7 @@ export function createSessionCacheEvaluator(deps: SessionCacheEvaluatorDeps): Go }; if (deps.apiKey) options.apiKey = deps.apiKey; if (deps.signal) options.signal = deps.signal; + if (deps.sessionId) options.sessionId = deps.sessionId; try { const response = await deps.completeSimple(deps.model, modelContext, options); @@ -302,7 +309,7 @@ export function createSessionCacheEvaluator(deps: SessionCacheEvaluatorDeps): Go } /** - * Create an LLM-based goal evaluator backed by a small, fast model. + * Create an LLM-based goal evaluator that sends a standalone transcript prompt. */ export function createLlmGoalEvaluator(deps: LlmEvaluatorDeps): GoalEvaluator { return { @@ -353,60 +360,25 @@ export function createLlmGoalEvaluator(deps: LlmEvaluatorDeps): GoalEvaluator { } /** - * Resolve a small, fast model to use for goal evaluation. - * - * This is the fallback path, used when the session prefix isn't available or - * didn't hit the cache. Without a cache to share there is nothing to gain - * from the session's model, so the cheapest authenticated text model wins: - * the standalone prompt would otherwise bill a full transcript at the - * session model's rate. - * - * If a model is configured (as `provider:modelId` or just `modelId`), it is - * tried first. Otherwise the registry is searched for the cheapest available - * text model that has configured auth. This avoids hardcoding Anthropic IDs - * and works with any provider the user has set up. + * Resolve the pinned `goal.evaluatorModel` (`provider:modelId` or `modelId`) + * to a model with usable auth. Returns undefined when it can't be used; the + * caller then falls back to the session's selected model. */ export async function resolveEvaluatorModel( registry: ModelRegistry, - configured?: string, + configured: string, ): Promise<{ model: Model; apiKey?: string } | undefined> { - let candidates: Model[] = []; - - if (configured) { - const [providerPart, idPart] = configured.includes(":") ? configured.split(":") : [undefined, configured]; - candidates = registry.getAll().filter((m) => { - if (providerPart && m.provider !== providerPart) return false; - return m.id === idPart; - }); - // A configured-but-unusable model shouldn't silently disable the - // evaluator; fall back to the cheapest authenticated text model. - candidates = candidates.concat(findSmallFastModels(registry)); - } else { - candidates = findSmallFastModels(registry); - } - + const [providerPart, idPart] = configured.includes(":") ? configured.split(":") : [undefined, configured]; + const candidates = registry.getAll().filter((m) => (!providerPart || m.provider === providerPart) && m.id === idPart); for (const model of candidates) { if (!registry.hasConfiguredAuth(model)) continue; const auth = await registry.getApiKeyAndHeaders(model); if (!auth.ok) continue; return { model, apiKey: auth.apiKey }; } - return undefined; } -function findSmallFastModels(registry: ModelRegistry): Model[] { - return registry - .getAll() - .filter((m) => m.input.includes("text") && !m.reasoning) - .sort((a, b) => { - const aCost = a.cost.input + a.cost.output; - const bCost = b.cost.input + b.cost.output; - if (aCost !== bCost) return aCost - bCost; - return a.contextWindow - b.contextWindow; - }); -} - /** * Simple deterministic evaluator: if any success keyword appears in the * latest turn text, the goal is met. diff --git a/packages/cli/src/extensions/goal/goal.integration.test.ts b/packages/cli/src/extensions/goal/goal.integration.test.ts index 892043c30..b4c5313e7 100644 --- a/packages/cli/src/extensions/goal/goal.integration.test.ts +++ b/packages/cli/src/extensions/goal/goal.integration.test.ts @@ -16,7 +16,7 @@ * network call via `mock.module` so no real API requests are made. */ import { afterAll, afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AssistantMessage, ToolResultMessage } from "@earendil-works/pi-ai"; @@ -31,7 +31,7 @@ import { _setGlobalConfigDir } from "../../config/io.js"; import { goalExtension } from "./index.js"; import { getGoal, getPendingGuidance, resetSession, formatGoalStatus, toMetaGoalStatus } from "./state.js"; import { parseLlmVerdict } from "./evaluator.js"; -import { isCacheReuseViable, resetSessionContext } from "./session-context.js"; +import { resetSessionContext } from "./session-context.js"; // ── Mock the network path for the LLM evaluator ────────────────────────────── @@ -120,6 +120,19 @@ function createFakePi(): FakePi { return { pi, handlers, messages, userMessages, entries, events, commands, tools }; } +const HAIKU_MODEL = { + provider: "anthropic", + id: "claude-haiku-4-5", + name: "Claude Haiku 4.5", + api: "anthropic-messages", + baseUrl: "", + reasoning: false, + input: ["text"], + cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200_000, + maxTokens: 4096, +}; + function createFakeCtx(overrides: { cwd?: string; entries?: SessionEntry[]; @@ -135,22 +148,11 @@ function createFakeCtx(overrides: { getEntries: () => overrides.entries ?? [], }, modelRegistry: { - getAll: () => [{ - provider: "anthropic", - id: "claude-haiku-4-5", - name: "Claude Haiku 4.5", - api: "anthropic-messages", - baseUrl: "", - reasoning: false, - input: ["text"], - cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 200_000, - maxTokens: 4096, - }], + getAll: () => [HAIKU_MODEL], hasConfiguredAuth: () => true, getApiKeyAndHeaders: async () => ({ ok: true as const, apiKey: "fake-api-key", headers: {} }), }, - model: overrides.model, + model: "model" in overrides ? overrides.model : HAIKU_MODEL, signal: overrides.signal ?? undefined, shutdown: overrides.shutdown ?? (() => {}), ui: { @@ -437,6 +439,57 @@ describe("/goal multi-turn integration", () => { ]) expect(parseLlmVerdict(JSON.stringify(verdict)).verdict).toBe("uncertain"); }); + test("goal_wait is only active while a goal is active", async () => { + const { pi, commands } = createFakePi(); + let active = ["bash"]; + (pi as any).getActiveTools = () => active; + (pi as any).setActiveTools = (names: string[]) => { active = names; }; + const ctx = createFakeCtx({ cwd: tmpCwd }); + goalExtension(pi); + + await commands.get("goal")!('"tests pass" --evaluator keyword --keyword "tests pass"', ctx as ExtensionCommandContext); + expect(active).toEqual(["bash", "goal_wait"]); + + await commands.get("goal")!("clear", ctx as ExtensionCommandContext); + expect(active).toEqual(["bash"]); + }); + + test("manual compaction resumes an active goal once the session is idle", async () => { + const { pi, handlers, commands, userMessages } = createFakePi(); + let idle = true; + let pending = false; + const ctx = Object.assign(createFakeCtx({ cwd: tmpCwd }), { + isIdle: () => idle, + hasPendingMessages: () => pending, + }); + goalExtension(pi); + await commands.get("goal")!('"tests pass" --evaluator keyword --keyword "tests pass"', ctx as ExtensionCommandContext); + const compact = async () => { + for (const h of handlers.get("session_compact") ?? []) await h({ type: "session_compact" }, ctx); + await new Promise((r) => setTimeout(r, 5)); + }; + + userMessages.length = 0; + await compact(); + expect(userMessages).toHaveLength(1); + expect(userMessages[0]!.content).toContain("tests pass"); + + // Mid-run or already-queued continuation: don't double-nudge. + userMessages.length = 0; + idle = false; + await compact(); + idle = true; + pending = true; + await compact(); + expect(userMessages).toHaveLength(0); + + // Waiting goals stay waiting. + pending = false; + await commands.get("goal")!("pause", ctx as ExtensionCommandContext); + await compact(); + expect(userMessages).toHaveLength(0); + }); + test("keyword evaluator loop: unmet → guidance → agent acts → goal cleared", async () => { const { pi, handlers, messages, entries, events, commands, userMessages } = createFakePi(); const shutdown = mock(() => {}); @@ -648,12 +701,26 @@ describe("/goal multi-turn integration", () => { expect(context.messages).toHaveLength(2); expect(context.messages[1]!.role).toBe("user"); - // The cache read was non-zero, so the path stays enabled. - expect(isCacheReuseViable("goal-integration-session")).toBe(true); expect(getGoal("goal-integration-session")?.evaluations.at(-1)?.verdict).toBe("not_met"); }); - test("falls back to a standalone cheap-model call once the cache misses", async () => { + test("judges on the session model without a captured prefix, and when a pinned model is unusable", async () => { + writeFileSync(join(tmpHome, "config.json"), JSON.stringify({ goal: { evaluatorModel: "nope:missing" } })); + const { pi, commands, handlers } = createFakePi(); + const sessionModel = { ...HAIKU_MODEL, id: "claude-sonnet-4-5", cost: { input: 3, output: 15, cacheRead: 0, cacheWrite: 0 } }; + const ctx = createFakeCtx({ cwd: tmpCwd, model: sessionModel }); + + goalExtension(pi); + await commands.get("goal")!('"services are green" --min-turns 1', ctx as ExtensionCommandContext); + // No `context` event fired, so there is no prefix to reuse. + await runTurnEnd(handlers, ctx, 1, "I restarted the services."); + + expect(fakeCompleteSimple).toHaveBeenCalledTimes(1); + const [model] = fakeCompleteSimple.mock.calls[0] as unknown as [{ id: string }]; + expect(model.id).toBe("claude-sonnet-4-5"); + }); + + test("keeps judging on the session model with the session context after a cache miss", async () => { const { pi, handlers, commands } = createFakePi(); const sessionModel = { provider: "anthropic", @@ -689,25 +756,20 @@ describe("/goal multi-turn integration", () => { })); await runTurnEnd(handlers, ctx, 1, "I restarted the services."); - expect(isCacheReuseViable("goal-integration-session")).toBe(false); - // Run 2: no longer worth re-sending the conversation at full price on - // the session model — fall back to the cheapest authenticated model - // with a standalone transcript. + // Run 2: a miss never swaps the judge to another model. for (const handler of handlers.get("context") ?? []) { await handler({ type: "context", messages: [{ role: "user", content: "check again", timestamp: 3 }] } as never, ctx); } await runTurnEnd(handlers, ctx, 2, "Still restarting."); expect(fakeCompleteSimple).toHaveBeenCalledTimes(2); - const [fallbackModel, fallbackContext] = fakeCompleteSimple.mock.calls[1] as unknown as [ + const [secondModel, secondContext] = fakeCompleteSimple.mock.calls[1] as unknown as [ { id: string }, { systemPrompt?: string; messages: Array<{ content: unknown }> }, ]; - expect(fallbackModel.id).toBe("claude-haiku-4-5"); - expect(fallbackContext.systemPrompt).toBeUndefined(); - expect(fallbackContext.messages).toHaveLength(1); - expect(fallbackContext.messages[0]!.content as string).toContain("Conversation so far:"); + expect(secondModel.id).toBe("claude-sonnet-4-5"); + expect(secondContext.systemPrompt).toBe("session system prompt"); }); test("goal is not cleared until the condition is actually met", async () => { diff --git a/packages/cli/src/extensions/goal/goal.test.ts b/packages/cli/src/extensions/goal/goal.test.ts index 51c5b1a85..360d3c16f 100644 --- a/packages/cli/src/extensions/goal/goal.test.ts +++ b/packages/cli/src/extensions/goal/goal.test.ts @@ -24,13 +24,7 @@ import { createSessionCacheEvaluator, resolveEvaluatorModel, } from "./evaluator.js"; -import { - captureSessionContext, - isCacheReuseViable, - recordCacheOutcome, - resetSessionContext, - toLlmMessages, -} from "./session-context.js"; +import { toLlmMessages } from "./session-context.js"; import { extractLatestTurnText, buildTranscript } from "./transcript.js"; import type { AssistantMessage, Context, Model, SimpleStreamOptions } from "@earendil-works/pi-ai"; import type { GoalState, GoalVerdict } from "./types.js"; @@ -555,50 +549,13 @@ describe("resolveEvaluatorModel", () => { expect(resolved?.model.id).toBe("gpt-4o-mini"); }); - test("falls back to the cheapest authenticated text model", async () => { - const registry = makeRegistry([ - { provider: "anthropic", id: "claude-opus", input: ["text"], cost: { input: 15, output: 75, cacheRead: 0, cacheWrite: 0 }, contextWindow: 200_000, maxTokens: 4096, reasoning: false, name: "Opus", api: "anthropic-messages", baseUrl: "" }, - { provider: "openai", id: "gpt-4o-mini", input: ["text"], cost: { input: 0.15, output: 0.6, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128_000, maxTokens: 4096, reasoning: false, name: "Mini", api: "openai-completions", baseUrl: "" }, - { provider: "groq", id: "llama-8b", input: ["text"], cost: { input: 0.05, output: 0.08, cacheRead: 0, cacheWrite: 0 }, contextWindow: 8_192, maxTokens: 4096, reasoning: false, name: "Llama", api: "openai-completions", baseUrl: "" }, - ]); - const resolved = await resolveEvaluatorModel(registry); - expect(resolved?.model.provider).toBe("groq"); - expect(resolved?.model.id).toBe("llama-8b"); - }); - - test("ignores models without configured auth", async () => { - const registry = makeRegistry([ - { provider: "unauthenticated", id: "cheap-model", input: ["text"], cost: { input: 0.01, output: 0.01, cacheRead: 0, cacheWrite: 0 }, contextWindow: 8_192, maxTokens: 4096, reasoning: false, name: "Cheap", api: "openai-completions", baseUrl: "" }, - { provider: "openai", id: "gpt-4o-mini", input: ["text"], cost: { input: 0.15, output: 0.6, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128_000, maxTokens: 4096, reasoning: false, name: "Mini", api: "openai-completions", baseUrl: "" }, - ]); - const resolved = await resolveEvaluatorModel(registry); - expect(resolved?.model.id).toBe("gpt-4o-mini"); - }); - - test("returns undefined when nothing is available", async () => { - const registry = makeRegistry([]); - const resolved = await resolveEvaluatorModel(registry); - expect(resolved).toBeUndefined(); - }); - - test("does not prefer an expensive session model — unconfigured resolves cheapest", async () => { - // The evaluator prompt shares no prefix with the session context, so - // there is no prompt cache to hit by reusing the session's model. - const registry = makeRegistry([ - { provider: "anthropic", id: "claude-opus", input: ["text"], cost: { input: 15, output: 75, cacheRead: 0, cacheWrite: 0 }, contextWindow: 200_000, maxTokens: 4096, reasoning: false, name: "Opus", api: "anthropic-messages", baseUrl: "" }, - { provider: "openai", id: "gpt-4o-mini", input: ["text"], cost: { input: 0.15, output: 0.6, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128_000, maxTokens: 4096, reasoning: false, name: "Mini", api: "openai-completions", baseUrl: "" }, - ]); - const resolved = await resolveEvaluatorModel(registry); - expect(resolved?.model.id).toBe("gpt-4o-mini"); - }); - - test("falls back to the cheapest text model when the configured model lacks auth", async () => { + test("returns undefined when the pinned model lacks auth or doesn't exist", async () => { const registry = makeRegistry([ { provider: "unauthenticated", id: "pinned", input: ["text"], cost: { input: 0.01, output: 0.01, cacheRead: 0, cacheWrite: 0 }, contextWindow: 8_192, maxTokens: 4096, reasoning: false, name: "Pinned", api: "openai-completions", baseUrl: "" }, { provider: "openai", id: "gpt-4o-mini", input: ["text"], cost: { input: 0.15, output: 0.6, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128_000, maxTokens: 4096, reasoning: false, name: "Mini", api: "openai-completions", baseUrl: "" }, ]); - const resolved = await resolveEvaluatorModel(registry, "unauthenticated:pinned"); - expect(resolved?.model.id).toBe("gpt-4o-mini"); + expect(await resolveEvaluatorModel(registry, "unauthenticated:pinned")).toBeUndefined(); + expect(await resolveEvaluatorModel(registry, "missing")).toBeUndefined(); }); }); @@ -816,6 +773,21 @@ describe("createSessionCacheEvaluator", () => { expect(feedback.cacheReadTokens).toBe(900); }); + test("forwards the session id so providers can key the prompt cache", async () => { + let seenOptions: SimpleStreamOptions | undefined; + const evaluator = createSessionCacheEvaluator({ + completeSimple: async (_model, _context, options) => { + seenOptions = options; + return makeResponse('{"verdict": "no", "reason": "x"}', 900); + }, + model: { id: "gpt", provider: "openai-codex" } as Model, + captured, + sessionId: "pi-session-1", + }); + await evaluator.evaluate(makeState(), { latestTurnText: "", transcript: "", history: [], turnCount: 1, tokenSpend: 0 }); + expect(seenOptions?.sessionId).toBe("pi-session-1"); + }); + test("does not resend the transcript — the cached conversation is the transcript", async () => { let seen: Context | undefined; const evaluator = createSessionCacheEvaluator({ @@ -904,31 +876,6 @@ describe("toLlmMessages", () => { }); }); -describe("session context cache tracking", () => { - test("a cache read keeps the path viable; a miss disables it once", () => { - resetSessionContext("cache-session"); - captureSessionContext("cache-session", { - systemPrompt: "sys", - messages: [{ role: "user", content: "hi", timestamp: 1 }], - tools: [], - capturedAt: 1, - }); - - expect(isCacheReuseViable("cache-session")).toBe(true); - expect(recordCacheOutcome("cache-session", 500)).toBe(false); - expect(isCacheReuseViable("cache-session")).toBe(true); - - // First miss disables reuse and reports the downgrade exactly once, - // so the warning isn't repeated every evaluation. - expect(recordCacheOutcome("cache-session", 0)).toBe(true); - expect(isCacheReuseViable("cache-session")).toBe(false); - expect(recordCacheOutcome("cache-session", 0)).toBe(false); - - resetSessionContext("cache-session"); - expect(isCacheReuseViable("cache-session")).toBe(true); - }); -}); - describe("pending guidance", () => { test("setPendingGuidance stores and getPendingGuidance retrieves", () => { resetSession("session-1"); diff --git a/packages/cli/src/extensions/goal/index.ts b/packages/cli/src/extensions/goal/index.ts index f247313fa..b399f5e20 100644 --- a/packages/cli/src/extensions/goal/index.ts +++ b/packages/cli/src/extensions/goal/index.ts @@ -71,8 +71,6 @@ import { import { captureSessionContext, getCapturedSessionContext, - isCacheReuseViable, - recordCacheOutcome, resetSessionContext, snapshotActiveTools, toLlmMessages, @@ -107,12 +105,27 @@ function emitGoalStatusChanged( } } +const GOAL_WAIT_TOOL = "goal_wait"; + +type ToolLoadoutApi = Partial>; + +/** Only expose goal_wait while a goal is active, so the model can't call it without one. */ +function syncGoalWaitTool(pi: ToolLoadoutApi, active: boolean): void { + if (!pi.getActiveTools || !pi.setActiveTools) return; + const tools = pi.getActiveTools(); + const has = tools.includes(GOAL_WAIT_TOOL); + if (active && !has) pi.setActiveTools([...tools, GOAL_WAIT_TOOL]); + else if (!active && has) pi.setActiveTools(tools.filter((t) => t !== GOAL_WAIT_TOOL)); +} + + function broadcastGoalStatus( sessionId: string, state: GoalState | undefined, ctx: ExtensionContext, - pi: Pick, + pi: Pick & ToolLoadoutApi, ): void { + syncGoalWaitTool(pi, state?.status === "active"); if (state?.status === "active") { ctx.ui.setStatus("goal", formatCompactGoalStatus(state)); emitGoalStatusChanged(pi, toMetaGoalStatus(state)); @@ -327,12 +340,11 @@ function steerContinuation( * failures degrade to an "uncertain" verdict (which never auto-continues) * rather than throwing out of the turn handler. * - * Preference order for the LLM evaluator: - * 1. Reuse the session's own context on the session's model, so the judge - * call reads the conversation from the provider's prompt cache. - * 2. A standalone prompt on the cheapest authenticated model, used when - * there is no captured prefix, when a model is explicitly pinned, or - * once a cache miss has proved the prefix doesn't match. + * The judge runs on the session's currently selected model unless + * `goal.evaluatorModel` pins another one. On the session model it reuses the + * session's own context so the conversation is read from the provider's + * prompt cache; a pinned model (or no captured prefix yet) gets a standalone + * transcript prompt instead. */ async function resolveEvaluator( sessionId: string, @@ -352,39 +364,45 @@ async function resolveEvaluator( }); try { - const captured = getCapturedSessionContext(sessionId); - const canReuseSession = - !config.evaluatorModel && - captured !== undefined && - captured.messages.length > 0 && - isCacheReuseViable(sessionId) && - ctx.model !== undefined && - ctx.modelRegistry.hasConfiguredAuth(ctx.model); - - if (canReuseSession && ctx.model) { - const auth = await ctx.modelRegistry.getApiKeyAndHeaders(ctx.model); - if (auth.ok) { - return createSessionCacheEvaluator({ + if (config.evaluatorModel) { + const pinned = await resolveEvaluatorModel(ctx.modelRegistry, config.evaluatorModel); + if (pinned) { + return createLlmGoalEvaluator({ completeSimple, - model: ctx.model, - apiKey: auth.apiKey, + model: pinned.model, + apiKey: pinned.apiKey, maxTokens: config.evaluatorMaxTokens, signal: ctx.signal, - captured: captured!, }); } + log.warn(`Pinned goal evaluator model ${config.evaluatorModel} is unavailable; using the session model.`); } - const resolved = await resolveEvaluatorModel(ctx.modelRegistry, config.evaluatorModel); - if (!resolved) { - const reason = "No configured evaluator model with auth is available; skipping LLM evaluation."; + const model = ctx.model; + if (!model) return uncertain("No session model is selected; skipping LLM evaluation."); + const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model); + if (!auth.ok) { + const reason = `Session model ${model.provider}/${model.id} has no usable auth; skipping LLM evaluation.`; log.warn(reason); return uncertain(reason); } + + const captured = getCapturedSessionContext(sessionId); + if (captured && captured.messages.length > 0) { + return createSessionCacheEvaluator({ + completeSimple, + model, + apiKey: auth.apiKey, + maxTokens: config.evaluatorMaxTokens, + signal: ctx.signal, + captured, + sessionId: ctx.sessionManager.getSessionId(), + }); + } return createLlmGoalEvaluator({ completeSimple, - model: resolved.model, - apiKey: resolved.apiKey, + model, + apiKey: auth.apiKey, maxTokens: config.evaluatorMaxTokens, signal: ctx.signal, }); @@ -483,15 +501,6 @@ async function runGoalStopCheck( } return; } - // Verify the shared-prefix assumption instead of trusting it. A miss - // means we just paid full input price on the session model, so this - // session stops reusing the session context from here on. - if (feedback.cacheReadTokens !== undefined && recordCacheOutcome(sessionId, feedback.cacheReadTokens)) { - log.warn( - "Goal evaluator reused the session context but the provider reported no cache read; " + - "falling back to a standalone evaluator call for the rest of this session.", - ); - } state = recordEvaluation(sessionId, feedback, pi) ?? state; } catch (err) { if (!stillCurrent() || ctx.signal?.aborted) return; @@ -547,7 +556,9 @@ async function runGoalStopCheck( export const goalExtension: ExtensionFactory = (pi) => { pi.registerTool({ - name: "goal_wait", + name: GOAL_WAIT_TOOL, + // Activated by syncGoalWaitTool only while a goal is active. + defaultActive: false, label: "Wait for goal progress", description: "Pause an active /goal's automatic continuation when no useful work remains until external input arrives. Supply why you are blocked and the expected wake-up event, then finish your response. Incoming user messages, child results, background completions, and triggers can resume work. This does not schedule a timer or subscribe to events: arrange a notification first, or ask the user for the missing input. Do not poll or repeat waiting messages.", parameters: Type.Object({ @@ -624,7 +635,6 @@ export const goalExtension: ExtensionFactory = (pi) => { const sessionId = getSessionId(ctx); const state = getGoal(sessionId); if (!state || state.status !== "active" || state.condition.evaluator !== "llm") return undefined; - if (!isCacheReuseViable(sessionId)) return undefined; captureSessionContext(sessionId, { systemPrompt: ctx.getSystemPrompt(), @@ -635,6 +645,25 @@ export const goalExtension: ExtensionFactory = (pi) => { return undefined; }); + // Manual /compact aborts the current run, and runGoalStopCheck ignores + // aborted runs, so nothing would restart the loop. Once compaction has + // settled, resume the goal if the session is idle with nothing queued. + // Mid-run (threshold/overflow) compactions leave the run going or a + // continuation queued, so the idle check skips them. + pi.on("session_compact", (_event, ctx) => { + const sessionId = getSessionId(ctx); + setTimeout(() => { + const state = getGoal(sessionId); + if (!state || state.status !== "active" || state.waiting) return; + if (!ctx.isIdle() || ctx.hasPendingMessages() || hasActiveSubagents()) return; + if (enforceBudget(sessionId, state, pi)) { + broadcastGoalStatus(sessionId, getGoal(sessionId), ctx, pi); + return; + } + steerContinuation(sessionId, state, pi); + }, 0); + }); + pi.on("turn_end", async (event, ctx) => { await runGoalStopCheck(event, ctx, pi); broadcastGoalStatus(getSessionId(ctx), getGoal(getSessionId(ctx)), ctx, pi); diff --git a/packages/cli/src/extensions/goal/session-context.ts b/packages/cli/src/extensions/goal/session-context.ts index 824eacdb4..0b6c07928 100644 --- a/packages/cli/src/extensions/goal/session-context.ts +++ b/packages/cli/src/extensions/goal/session-context.ts @@ -10,11 +10,6 @@ * `tools → system → messages`, in that order. A tools block that differs by * one entry (or one position) invalidates everything after it, so the numbers * below come from what pi actually sent, not from what we think it sent. - * - * Even so, a match is never assumed. `recordCacheOutcome` tracks whether the - * provider reported a cache read; the evaluator stops using this path when it - * doesn't, so a mismatch degrades to a cheap standalone call instead of - * silently billing full-price transcripts against the session model. */ import type { AgentMessage } from "@earendil-works/pi-agent-core"; import type { Message, Tool } from "@earendil-works/pi-ai"; @@ -33,14 +28,6 @@ export interface CapturedSessionContext { const capturedBySessionId = new Map(); -/** - * Sessions where a cache-reusing evaluation ran but the provider reported no - * cache read. Once a session lands here the evaluator stops reusing the - * session context, because a miss means we would be re-sending the whole - * conversation at full input price on the session's (often expensive) model. - */ -const cacheMissBySessionId = new Set(); - /** Replace the captured context for a session. Only the latest is kept. */ export function captureSessionContext(sessionId: string, captured: CapturedSessionContext): void { capturedBySessionId.set(sessionId, captured); @@ -50,31 +37,9 @@ export function getCapturedSessionContext(sessionId: string): CapturedSessionCon return capturedBySessionId.get(sessionId); } -/** - * Whether the cache-reusing evaluator path is still worth trying for this - * session. - */ -export function isCacheReuseViable(sessionId: string): boolean { - return !cacheMissBySessionId.has(sessionId); -} - -/** - * Record whether a cache-reusing evaluation actually read from the cache. - * - * Returns true when this call flipped the session to "not viable", so the - * caller can log the downgrade exactly once. - */ -export function recordCacheOutcome(sessionId: string, cacheReadTokens: number | undefined): boolean { - if ((cacheReadTokens ?? 0) > 0) return false; - if (cacheMissBySessionId.has(sessionId)) return false; - cacheMissBySessionId.add(sessionId); - return true; -} - /** Drop all captured state for a session. */ export function resetSessionContext(sessionId: string): void { capturedBySessionId.delete(sessionId); - cacheMissBySessionId.delete(sessionId); } /** @@ -87,9 +52,8 @@ export function resetSessionContext(sessionId: string): void { * * Approximates, not reproduces: a session whose `convertToLlm` rewrites a * custom message into a real one (rather than dropping it) will produce a - * prefix that differs from what pi sent, and the cache won't hit. That is - * exactly what `recordCacheOutcome` is for — the miss is detected on the - * first evaluation and the session stops using this path. + * prefix that differs from what pi sent, and the cache won't hit; the judge + * call still works, it just pays full input price. */ export function toLlmMessages(messages: AgentMessage[]): Message[] { return messages.filter( diff --git a/packages/docs/src/content/docs/customization/configuration.mdx b/packages/docs/src/content/docs/customization/configuration.mdx index c0708d10a..b4b726095 100644 --- a/packages/docs/src/content/docs/customization/configuration.mdx +++ b/packages/docs/src/content/docs/customization/configuration.mdx @@ -197,7 +197,7 @@ A documented subset of settings can also be overridden with **environment variab | `subagent.maxConcurrency` | `number` | `4` | Max concurrent agent sessions within one parallel call. **Global config only.** | | `subagent.backgroundAfterSeconds` | — | — | Not supported. Use `bash.backgroundAfterSeconds` for shell auto-backgrounding. | | `subagent.keepLoadedTools` | — | — | Not supported. Use `toolSearch.keepLoadedTools` for deferred MCP tools. | -| `goal.evaluatorModel` | `string` | the session's own model | Advanced: pin a model (`provider:modelId` or `modelId`) for the `/goal` LLM evaluator. Opts out of session-context cache reuse; falls back to the cheapest authenticated text model if the pinned model has no credentials. | +| `goal.evaluatorModel` | `string` | the session's own model | Advanced: pin a model (`provider:modelId` or `modelId`) for the `/goal` LLM evaluator. Opts out of session-context cache reuse; falls back to the session's model if the pinned model has no credentials. | | `goal.evaluatorMaxTokens` | `number` | `512` | Maximum output tokens for each `/goal` evaluator call. | | `goal.evaluateEveryNTurns` | `number` | `1` | Default cadence for the LLM evaluator, in completed agent runs. `1` = every run. Ignored by the keyword evaluator. | | `goal.minTurnsBeforeEvaluate` | `number` | `1` | Minimum completed agent runs before the goal evaluator runs for the first time. | diff --git a/packages/docs/src/content/docs/features/goals.mdx b/packages/docs/src/content/docs/features/goals.mdx index d5ec1debc..e3abdab6b 100644 --- a/packages/docs/src/content/docs/features/goals.mdx +++ b/packages/docs/src/content/docs/features/goals.mdx @@ -65,8 +65,8 @@ The LLM evaluator is told to judge on **evidence** — command output, test resu Rather than assembling a separate transcript, the evaluator re-sends the exact context your session just sent — same system prompt, same tools, same messages — with the judge question appended as one final user message. Providers cache on an exact prefix, so that conversation is served from cache and only the appended question is billed as new input. It also means the judge reads the real conversation at full fidelity, and nothing is shared with a provider your session wasn't already using. -