Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 101 additions & 1 deletion src/main/core/agent/__tests__/research-loop-guard.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";

import { ResearchLoopGuard } from "../research-loop-guard";
import { ResearchLoopGuard, WebSearchRunGuard } from "../research-loop-guard";

const call = (toolName: string, args: unknown, index: number) => ({
toolName,
Expand Down Expand Up @@ -193,3 +193,103 @@ describe("ResearchLoopGuard", () => {
});
});
});

describe("WebSearchRunGuard", () => {
it("blocks an exact retry after a failed search and removes the tool", () => {
const guard = new WebSearchRunGuard();
const first = call("webSearch", { query: "latest news" }, 1);

expect(guard.beforeToolCall(first)).toEqual({ allow: true });
guard.observeToolResult({
...first,
rawOutput: {
success: false,
retryable: false,
error: "Tavily 400 Bad Request",
},
});

expect(
guard.beforeToolCall(call("webSearch", { query: "latest news" }, 2)),
).toMatchObject({
allow: false,
result: {
success: false,
blocked: true,
retryable: false,
},
});
expect(
guard.getStepPolicy(["readFile", "webSearch", "webFetch"]),
).toMatchObject({
activeTools: ["readFile", "webFetch"],
toolChoice: "auto",
message: expect.stringMatching(/disabled|failed|continue/i),
});
});

it("opens the circuit after two consecutive failed searches", () => {
const guard = new WebSearchRunGuard();

for (let index = 1; index <= 2; index++) {
const toolCall = call(
"webSearch",
{ query: `different query ${index}` },
index,
);
expect(guard.beforeToolCall(toolCall)).toEqual({ allow: true });
guard.observeToolResult({
...toolCall,
rawOutput: {
success: false,
retryable: index === 1,
error: `failure ${index}`,
},
});
}

expect(
guard.beforeToolCall(
call("webSearch", { query: "third different query" }, 3),
),
).toMatchObject({
allow: false,
result: {
success: false,
blocked: true,
},
});
});

it("allows four searches and disables further search calls", () => {
const guard = new WebSearchRunGuard();

for (let index = 1; index <= 4; index++) {
const toolCall = call(
"webSearch",
{ query: `focused query ${index}` },
index,
);
expect(guard.beforeToolCall(toolCall)).toEqual({ allow: true });
guard.observeToolResult({
...toolCall,
rawOutput: {
results: [{ url: `https://example.com/${index}` }],
},
});
}

expect(guard.getStepPolicy(["webSearch", "webFetch"])).toMatchObject({
activeTools: ["webFetch"],
message: expect.stringMatching(/budget|four|4/i),
});
expect(
guard.beforeToolCall(
call("webSearch", { query: "one search too many" }, 5),
),
).toMatchObject({
allow: false,
result: { success: false, blocked: true },
});
});
});
128 changes: 121 additions & 7 deletions src/main/core/agent/__tests__/web-search.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,12 @@ describe("buildWebSearchTool", () => {
resolveTavilyToken: async () => null,
});
const out = (await tool.execute({ query: "anything" }, fakeCtx())) as {
success?: boolean;
error?: string;
retryable?: boolean;
};
expect(out.success).toBe(false);
expect(out.retryable).toBe(false);
expect(out.error).toMatch(/No Tavily API key/i);
expect(fetchImpl).not.toHaveBeenCalled();
});
Expand Down Expand Up @@ -130,7 +134,30 @@ describe("buildWebSearchTool", () => {
expect(value.length).toBeLessThan(10_000);
});

it("passes Tavily recency params through to the API body (topic/days/timeRange/dates)", async () => {
it("projects structured search failures to the model", async () => {
const tool = buildWebSearchTool({
fetchImpl: vi.fn() as never,
resolveTavilyToken: async () => "tvly-test",
});
const projected = await tool.toModelOutput?.({
toolCallId: "search-error",
input: { query: "latest news" },
output: {
success: false,
status: 400,
retryable: false,
error: "Tavily 400 Bad Request: invalid filters",
hint: "Correct the parameters; do not retry the same call.",
},
});

const value = (projected as { value: string }).value;
expect(value).toContain("Error: Tavily 400 Bad Request");
expect(value).toContain("Retryable: false");
expect(value).toContain("Correct the parameters");
});

it("prefers absolute dates over relative recency filters", async () => {
const fetchImpl = vi.fn(async () =>
jsonResponse({ answer: null, results: [] }),
);
Expand All @@ -155,10 +182,10 @@ describe("buildWebSearchTool", () => {
];
const body = JSON.parse(init.body as string) as Record<string, unknown>;
expect(body.topic).toBe("news");
expect(body.days).toBe(1);
expect(body.time_range).toBe("day");
expect(body.start_date).toBe("2026-05-18");
expect(body.end_date).toBe("2026-05-19");
expect(body).not.toHaveProperty("days");
expect(body).not.toHaveProperty("time_range");
});

it("does not include recency params when not provided (backward compatible)", async () => {
Expand All @@ -182,17 +209,104 @@ describe("buildWebSearchTool", () => {
expect(body).not.toHaveProperty("end_date");
});

it("throws on non-2xx Tavily response", async () => {
it("returns a structured non-retryable error for Tavily 400 responses", async () => {
const fetchImpl = vi.fn(
async () =>
new Response("rate limit", { status: 429, statusText: "Too Many" }),
new Response("invalid date filters", {
status: 400,
statusText: "Bad Request",
}),
);
const tool = buildWebSearchTool({
fetchImpl: fetchImpl as never,
resolveTavilyToken: async () => "tvly-test",
});
await expect(tool.execute({ query: "x" }, fakeCtx())).rejects.toThrow(
/Tavily 429/,
await expect(tool.execute({ query: "x" }, fakeCtx())).resolves.toEqual(
expect.objectContaining({
success: false,
status: 400,
retryable: false,
error: expect.stringMatching(/Tavily 400.*invalid date filters/),
}),
);
});

it("retries a transient Tavily failure once inside the tool", async () => {
vi.useFakeTimers();
try {
const fetchImpl = vi
.fn()
.mockResolvedValueOnce(
new Response("temporary outage", {
status: 503,
statusText: "Unavailable",
}),
)
.mockResolvedValueOnce(
jsonResponse({
answer: "Recovered",
results: [],
}),
);
const tool = buildWebSearchTool({
fetchImpl: fetchImpl as never,
resolveTavilyToken: async () => "tvly-test",
});

const resultPromise = tool.execute({ query: "x" }, fakeCtx());
await vi.runAllTimersAsync();

await expect(resultPromise).resolves.toEqual(
expect.objectContaining({ answer: "Recovered" }),
);
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
vi.useRealTimers();
}
});

it("removes the abort listener after the retry delay completes", async () => {
vi.useFakeTimers();
try {
const controller = new AbortController();
const addEventListener = vi.spyOn(controller.signal, "addEventListener");
const removeEventListener = vi.spyOn(
controller.signal,
"removeEventListener",
);
const fetchImpl = vi
.fn()
.mockResolvedValueOnce(
new Response("temporary outage", {
status: 503,
statusText: "Unavailable",
}),
)
.mockResolvedValueOnce(
jsonResponse({
answer: "Recovered",
results: [],
}),
);
const tool = buildWebSearchTool({
fetchImpl: fetchImpl as never,
resolveTavilyToken: async () => "tvly-test",
});

const resultPromise = tool.execute(
{ query: "x" },
{ ...fakeCtx(), signal: controller.signal },
);
await vi.runAllTimersAsync();
await resultPromise;

const abortListener = addEventListener.mock.calls.find(
([type]) => type === "abort",
)?.[1];
expect(abortListener).toBeDefined();
expect(removeEventListener).toHaveBeenCalledWith("abort", abortListener);
} finally {
vi.useRealTimers();
}
});
});
92 changes: 92 additions & 0 deletions src/main/core/agent/research-loop-guard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,98 @@ const phaseTransitionResult = (
nextAction,
});

const failedToolOutput = (rawOutput: unknown): boolean => {
if (!rawOutput || typeof rawOutput !== "object") return false;
const output = rawOutput as { success?: unknown; error?: unknown };
return output.success === false || Boolean(output.error);
};

const blockedSearchResult = (reason: string) => ({
success: false,
blocked: true,
retryable: false,
error: reason,
hint: "Web search is disabled for the rest of this run. Continue from existing evidence or explain the limitation.",
});

export interface WebSearchRunGuardConfig {
maxCalls: number;
maxConsecutiveFailures: number;
}

export const DEFAULT_WEB_SEARCH_RUN_GUARD_CONFIG: WebSearchRunGuardConfig = {
maxCalls: 4,
maxConsecutiveFailures: 2,
};

/** Per-run guard for bounded web search on the main chat path. */
export class WebSearchRunGuard {
private readonly config: WebSearchRunGuardConfig;
private readonly failedCallSignatures = new Set<string>();
private calls = 0;
private consecutiveFailures = 0;
private blockedReason = "";

constructor(config: Partial<WebSearchRunGuardConfig> = {}) {
this.config = { ...DEFAULT_WEB_SEARCH_RUN_GUARD_CONFIG, ...config };
}

beforeToolCall(call: {
toolName: string;
toolCallId: string;
args: unknown;
}): BeforeToolCallDecision {
if (call.toolName !== WEB_SEARCH_TOOL) return { allow: true };
const signature = callSignature(call.toolName, call.args);
if (this.failedCallSignatures.has(signature)) {
this.blockedReason =
"Web search was disabled because the same failed call was repeated.";
}
if (this.blockedReason) {
return {
allow: false,
reason: this.blockedReason,
result: blockedSearchResult(this.blockedReason),
};
}
if (this.calls >= this.config.maxCalls) {
this.blockedReason = `Focused web search budget reached (${this.config.maxCalls} calls).`;
return {
allow: false,
reason: this.blockedReason,
result: blockedSearchResult(this.blockedReason),
};
}
this.calls++;
if (this.calls >= this.config.maxCalls) {
this.blockedReason = `Focused web search budget reached (${this.config.maxCalls} calls).`;
}
return { allow: true };
}

observeToolResult(result: ToolObservedResult): void {
if (result.toolName !== WEB_SEARCH_TOOL) return;
if (!failedToolOutput(result.rawOutput)) {
this.consecutiveFailures = 0;
return;
}
this.failedCallSignatures.add(callSignature(result.toolName, result.args));
this.consecutiveFailures++;
if (this.consecutiveFailures >= this.config.maxConsecutiveFailures) {
this.blockedReason = `Web search was disabled after ${this.config.maxConsecutiveFailures} consecutive failed searches.`;
}
}

getStepPolicy(availableTools: string[]): ResearchStepPolicy | undefined {
if (!this.blockedReason) return undefined;
return {
activeTools: availableTools.filter((tool) => tool !== WEB_SEARCH_TOOL),
toolChoice: "auto",
message: `${this.blockedReason} Continue from the evidence already collected or explain the limitation; do not attempt another web search.`,
};
}
}

/** Per-run research guard: bounded calls, exact-loop detection, and source novelty. */
export class ResearchLoopGuard {
private readonly config: ResearchLoopGuardConfig;
Expand Down
Loading
Loading