Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 15 additions & 43 deletions packages/cli/src/extensions/goal/evaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* - `keywordGoalEvaluator`: fast, local keyword check.
* - `createLlmGoalEvaluator(...)`: sends a compact transcript + goal to a
* small, fast model (default Anthropic Haiku) and parses a yes/no decision.
* model (the session's, or a pinned one) and parses a yes/no decision.
*/
import type {
AssistantMessage,
Expand Down Expand Up @@ -199,6 +199,12 @@ export interface LlmEvaluatorDeps {
export interface SessionCacheEvaluatorDeps extends LlmEvaluatorDeps {
/** The session prefix to reuse, exactly as pi last sent it. */
captured: CapturedSessionContext;
/**
* The id pi sends as `sessionId` on the session's own calls. Providers key
* their prompt cache on it (openai-codex `prompt_cache_key`); without it
* the judge call always misses and the evaluator downgrades for good.
*/
sessionId?: string;
}

/**
Expand Down Expand Up @@ -270,6 +276,7 @@ export function createSessionCacheEvaluator(deps: SessionCacheEvaluatorDeps): Go
};
if (deps.apiKey) options.apiKey = deps.apiKey;
if (deps.signal) options.signal = deps.signal;
if (deps.sessionId) options.sessionId = deps.sessionId;

try {
const response = await deps.completeSimple(deps.model, modelContext, options);
Expand Down Expand Up @@ -302,7 +309,7 @@ export function createSessionCacheEvaluator(deps: SessionCacheEvaluatorDeps): Go
}

/**
* Create an LLM-based goal evaluator backed by a small, fast model.
* Create an LLM-based goal evaluator that sends a standalone transcript prompt.
*/
export function createLlmGoalEvaluator(deps: LlmEvaluatorDeps): GoalEvaluator {
return {
Expand Down Expand Up @@ -353,60 +360,25 @@ export function createLlmGoalEvaluator(deps: LlmEvaluatorDeps): GoalEvaluator {
}

/**
* Resolve a small, fast model to use for goal evaluation.
*
* This is the fallback path, used when the session prefix isn't available or
* didn't hit the cache. Without a cache to share there is nothing to gain
* from the session's model, so the cheapest authenticated text model wins:
* the standalone prompt would otherwise bill a full transcript at the
* session model's rate.
*
* If a model is configured (as `provider:modelId` or just `modelId`), it is
* tried first. Otherwise the registry is searched for the cheapest available
* text model that has configured auth. This avoids hardcoding Anthropic IDs
* and works with any provider the user has set up.
* Resolve the pinned `goal.evaluatorModel` (`provider:modelId` or `modelId`)
* to a model with usable auth. Returns undefined when it can't be used; the
* caller then falls back to the session's selected model.
*/
export async function resolveEvaluatorModel(
registry: ModelRegistry,
configured?: string,
configured: string,
): Promise<{ model: Model<any>; apiKey?: string } | undefined> {
let candidates: Model<any>[] = [];

if (configured) {
const [providerPart, idPart] = configured.includes(":") ? configured.split(":") : [undefined, configured];
candidates = registry.getAll().filter((m) => {
if (providerPart && m.provider !== providerPart) return false;
return m.id === idPart;
});
// A configured-but-unusable model shouldn't silently disable the
// evaluator; fall back to the cheapest authenticated text model.
candidates = candidates.concat(findSmallFastModels(registry));
} else {
candidates = findSmallFastModels(registry);
}

const [providerPart, idPart] = configured.includes(":") ? configured.split(":") : [undefined, configured];
const candidates = registry.getAll().filter((m) => (!providerPart || m.provider === providerPart) && m.id === idPart);
for (const model of candidates) {
if (!registry.hasConfiguredAuth(model)) continue;
const auth = await registry.getApiKeyAndHeaders(model);
if (!auth.ok) continue;
return { model, apiKey: auth.apiKey };
}

return undefined;
}

function findSmallFastModels(registry: ModelRegistry): Model<any>[] {
return registry
.getAll()
.filter((m) => m.input.includes("text") && !m.reasoning)
.sort((a, b) => {
const aCost = a.cost.input + a.cost.output;
const bCost = b.cost.input + b.cost.output;
if (aCost !== bCost) return aCost - bCost;
return a.contextWindow - b.contextWindow;
});
}

/**
* Simple deterministic evaluator: if any success keyword appears in the
* latest turn text, the goal is met.
Expand Down
116 changes: 89 additions & 27 deletions packages/cli/src/extensions/goal/goal.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* network call via `mock.module` so no real API requests are made.
*/
import { afterAll, afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import { mkdtempSync } from "node:fs";
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AssistantMessage, ToolResultMessage } from "@earendil-works/pi-ai";
Expand All @@ -31,7 +31,7 @@ import { _setGlobalConfigDir } from "../../config/io.js";
import { goalExtension } from "./index.js";
import { getGoal, getPendingGuidance, resetSession, formatGoalStatus, toMetaGoalStatus } from "./state.js";
import { parseLlmVerdict } from "./evaluator.js";
import { isCacheReuseViable, resetSessionContext } from "./session-context.js";
import { resetSessionContext } from "./session-context.js";

// ── Mock the network path for the LLM evaluator ──────────────────────────────

Expand Down Expand Up @@ -120,6 +120,19 @@ function createFakePi(): FakePi {
return { pi, handlers, messages, userMessages, entries, events, commands, tools };
}

const HAIKU_MODEL = {
provider: "anthropic",
id: "claude-haiku-4-5",
name: "Claude Haiku 4.5",
api: "anthropic-messages",
baseUrl: "",
reasoning: false,
input: ["text"],
cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200_000,
maxTokens: 4096,
};

function createFakeCtx(overrides: {
cwd?: string;
entries?: SessionEntry[];
Expand All @@ -135,22 +148,11 @@ function createFakeCtx(overrides: {
getEntries: () => overrides.entries ?? [],
},
modelRegistry: {
getAll: () => [{
provider: "anthropic",
id: "claude-haiku-4-5",
name: "Claude Haiku 4.5",
api: "anthropic-messages",
baseUrl: "",
reasoning: false,
input: ["text"],
cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200_000,
maxTokens: 4096,
}],
getAll: () => [HAIKU_MODEL],
hasConfiguredAuth: () => true,
getApiKeyAndHeaders: async () => ({ ok: true as const, apiKey: "fake-api-key", headers: {} }),
},
model: overrides.model,
model: "model" in overrides ? overrides.model : HAIKU_MODEL,
signal: overrides.signal ?? undefined,
shutdown: overrides.shutdown ?? (() => {}),
ui: {
Expand Down Expand Up @@ -437,6 +439,57 @@ describe("/goal multi-turn integration", () => {
]) expect(parseLlmVerdict(JSON.stringify(verdict)).verdict).toBe("uncertain");
});

test("goal_wait is only active while a goal is active", async () => {
const { pi, commands } = createFakePi();
let active = ["bash"];
(pi as any).getActiveTools = () => active;
(pi as any).setActiveTools = (names: string[]) => { active = names; };
const ctx = createFakeCtx({ cwd: tmpCwd });
goalExtension(pi);

await commands.get("goal")!('"tests pass" --evaluator keyword --keyword "tests pass"', ctx as ExtensionCommandContext);
expect(active).toEqual(["bash", "goal_wait"]);

await commands.get("goal")!("clear", ctx as ExtensionCommandContext);
expect(active).toEqual(["bash"]);
});

test("manual compaction resumes an active goal once the session is idle", async () => {
const { pi, handlers, commands, userMessages } = createFakePi();
let idle = true;
let pending = false;
const ctx = Object.assign(createFakeCtx({ cwd: tmpCwd }), {
isIdle: () => idle,
hasPendingMessages: () => pending,
});
goalExtension(pi);
await commands.get("goal")!('"tests pass" --evaluator keyword --keyword "tests pass"', ctx as ExtensionCommandContext);
const compact = async () => {
for (const h of handlers.get("session_compact") ?? []) await h({ type: "session_compact" }, ctx);
await new Promise((r) => setTimeout(r, 5));
};

userMessages.length = 0;
await compact();
expect(userMessages).toHaveLength(1);
expect(userMessages[0]!.content).toContain("tests pass");

// Mid-run or already-queued continuation: don't double-nudge.
userMessages.length = 0;
idle = false;
await compact();
idle = true;
pending = true;
await compact();
expect(userMessages).toHaveLength(0);

// Waiting goals stay waiting.
pending = false;
await commands.get("goal")!("pause", ctx as ExtensionCommandContext);
await compact();
expect(userMessages).toHaveLength(0);
});

test("keyword evaluator loop: unmet → guidance → agent acts → goal cleared", async () => {
const { pi, handlers, messages, entries, events, commands, userMessages } = createFakePi();
const shutdown = mock(() => {});
Expand Down Expand Up @@ -648,12 +701,26 @@ describe("/goal multi-turn integration", () => {
expect(context.messages).toHaveLength(2);
expect(context.messages[1]!.role).toBe("user");

// The cache read was non-zero, so the path stays enabled.
expect(isCacheReuseViable("goal-integration-session")).toBe(true);
expect(getGoal("goal-integration-session")?.evaluations.at(-1)?.verdict).toBe("not_met");
});

test("falls back to a standalone cheap-model call once the cache misses", async () => {
test("judges on the session model without a captured prefix, and when a pinned model is unusable", async () => {
writeFileSync(join(tmpHome, "config.json"), JSON.stringify({ goal: { evaluatorModel: "nope:missing" } }));
const { pi, commands, handlers } = createFakePi();
const sessionModel = { ...HAIKU_MODEL, id: "claude-sonnet-4-5", cost: { input: 3, output: 15, cacheRead: 0, cacheWrite: 0 } };
const ctx = createFakeCtx({ cwd: tmpCwd, model: sessionModel });

goalExtension(pi);
await commands.get("goal")!('"services are green" --min-turns 1', ctx as ExtensionCommandContext);
// No `context` event fired, so there is no prefix to reuse.
await runTurnEnd(handlers, ctx, 1, "I restarted the services.");

expect(fakeCompleteSimple).toHaveBeenCalledTimes(1);
const [model] = fakeCompleteSimple.mock.calls[0] as unknown as [{ id: string }];
expect(model.id).toBe("claude-sonnet-4-5");
});

test("keeps judging on the session model with the session context after a cache miss", async () => {
const { pi, handlers, commands } = createFakePi();
const sessionModel = {
provider: "anthropic",
Expand Down Expand Up @@ -689,25 +756,20 @@ describe("/goal multi-turn integration", () => {
}));

await runTurnEnd(handlers, ctx, 1, "I restarted the services.");
expect(isCacheReuseViable("goal-integration-session")).toBe(false);

// Run 2: no longer worth re-sending the conversation at full price on
// the session model — fall back to the cheapest authenticated model
// with a standalone transcript.
// Run 2: a miss never swaps the judge to another model.
for (const handler of handlers.get("context") ?? []) {
await handler({ type: "context", messages: [{ role: "user", content: "check again", timestamp: 3 }] } as never, ctx);
}
await runTurnEnd(handlers, ctx, 2, "Still restarting.");

expect(fakeCompleteSimple).toHaveBeenCalledTimes(2);
const [fallbackModel, fallbackContext] = fakeCompleteSimple.mock.calls[1] as unknown as [
const [secondModel, secondContext] = fakeCompleteSimple.mock.calls[1] as unknown as [
{ id: string },
{ systemPrompt?: string; messages: Array<{ content: unknown }> },
];
expect(fallbackModel.id).toBe("claude-haiku-4-5");
expect(fallbackContext.systemPrompt).toBeUndefined();
expect(fallbackContext.messages).toHaveLength(1);
expect(fallbackContext.messages[0]!.content as string).toContain("Conversation so far:");
expect(secondModel.id).toBe("claude-sonnet-4-5");
expect(secondContext.systemPrompt).toBe("session system prompt");
});

test("goal is not cleared until the condition is actually met", async () => {
Expand Down
Loading
Loading