diff --git a/packages/processor/src/__tests__/shared.test.ts b/packages/processor/src/__tests__/shared.test.ts index 8a1ba868..92208441 100644 --- a/packages/processor/src/__tests__/shared.test.ts +++ b/packages/processor/src/__tests__/shared.test.ts @@ -3,10 +3,12 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { + AgentPolicyRefusalError, attributionHeaders, buildInvestigateFieldRepairPrompt, buildInvestigateJsonRepairPrompt, buildRevalidateJsonRepairPrompt, + classifyPolicyRefusal, classifyQuotaError, formatJsonRepairFailureDebugText, isGatewayBaseUrl, @@ -201,6 +203,50 @@ describe("QuotaExhaustedError", () => { }); }); +describe("classifyPolicyRefusal", () => { + it("recognizes the Claude Code usage-policy refusal (issue #92)", () => { + const text = + "API Error: Claude Code is unable to respond to this request, which appears to violate our Usage Policy (https://www.anthropic.com/legal/aup). Try rephrasing the request or attempting a different approach. Request ID: req_123"; + expect(classifyPolicyRefusal(text)).toBe("anthropic-usage-policy"); + }); + + it("recognizes OpenAI safety-system rejections", () => { + expect(classifyPolicyRefusal('{"error":{"code":"invalid_prompt"}}')).toBe( + "openai-safety-system", + ); + expect(classifyPolicyRefusal("The prompt was rejected as a result of our safety system.")).toBe( + "openai-safety-system", + ); + }); + + it("recognizes generic content-policy envelopes", () => { + expect(classifyPolicyRefusal('{"error":{"type":"content_policy_violation"}}')).toBe( + "content-policy", + ); + }); + + it("returns undefined for ordinary malformed output and prose", () => { + // Strict on purpose: relabeling broken JSON as a refusal would report + // zero coverage where the real problem is formatting — worse than the + // status quo. Misses degrade to the existing fail-loud parse error. + expect(classifyPolicyRefusal("not JSON at all")).toBeUndefined(); + expect(classifyPolicyRefusal('```json\n[{"filePath":')).toBeUndefined(); + expect(classifyPolicyRefusal("")).toBeUndefined(); + expect(classifyPolicyRefusal("You've hit your usage limit.")).toBeUndefined(); + }); +}); + +describe("AgentPolicyRefusalError", () => { + it("carries source + raw, and the message includes both", () => { + const e = new AgentPolicyRefusalError("anthropic-usage-policy", "API Error: Claude Code…"); + expect(e.source).toBe("anthropic-usage-policy"); + expect(e.rawMessage).toContain("API Error"); + expect(e.message).toContain("anthropic-usage-policy"); + expect(e.name).toBe("AgentPolicyRefusalError"); + expect(e instanceof Error).toBe(true); + }); +}); + describe("isUsingAiGateway", () => { // We mutate process.env directly — capture and restore so tests don't // bleed environment into each other. @@ -403,6 +449,28 @@ describe("parseInvestigateResults", () => { ); }); + it("throws AgentPolicyRefusalError (not a parse error) on a provider policy refusal", () => { + // A refusal must never reach jsonrepair or surface as a formatting + // problem: the batch has zero coverage, and the repair follow-up would + // accept an empty findings array as a valid answer. + const refusal = + "API Error: Claude Code is unable to respond to this request, which appears to violate our Usage Policy (https://www.anthropic.com/legal/aup). Try rephrasing the request or attempting a different approach."; + expect(() => parseInvestigateResults(refusal, batch)).toThrow(AgentPolicyRefusalError); + }); + + it("does NOT flag valid findings JSON that quotes refusal text as evidence", () => { + // Classification runs only inside the JSON.parse catch, so a finding + // citing a refusal string parses like any other. + const finding = validFinding({ + description: + "Endpoint relays the upstream string 'unable to respond to this request, which appears to violate our Usage Policy (https://www.anthropic.com/legal/aup)' verbatim to clients.", + }); + const text = `\`\`\`json\n[{"filePath":"a.ts","findings":[${JSON.stringify(finding)}]}]\n\`\`\``; + const out = parseInvestigateResults(text, batch); + expect(out.results.find((r) => r.filePath === "a.ts")?.findings).toHaveLength(1); + expect(out.invalid).toEqual([]); + }); + it("salvages valid findings and reports invalid ones instead of dropping the file", () => { // One bad enum used to poison the whole file record downstream: the // read-side schema rejected the record and every valid finding in it @@ -579,6 +647,12 @@ describe("parseRevalidateVerdicts", () => { it("throws on parse failure", () => { expect(() => parseRevalidateVerdicts("garbage")).toThrow(/wasn't parseable JSON/); }); + + it("throws AgentPolicyRefusalError on a provider policy refusal", () => { + const refusal = + "API Error: Claude Code is unable to respond to this request, which appears to violate our Usage Policy (https://www.anthropic.com/legal/aup)."; + expect(() => parseRevalidateVerdicts(refusal)).toThrow(AgentPolicyRefusalError); + }); }); describe("writeParseFailureDebug", () => { diff --git a/packages/processor/src/agents/claude-agent-sdk.ts b/packages/processor/src/agents/claude-agent-sdk.ts index 0539444a..93dc6a3b 100644 --- a/packages/processor/src/agents/claude-agent-sdk.ts +++ b/packages/processor/src/agents/claude-agent-sdk.ts @@ -1,6 +1,7 @@ import { query, type SandboxSettings } from "@anthropic-ai/claude-agent-sdk"; import type { RefusalReport } from "@deepsec/core"; import { + AgentPolicyRefusalError, attributionHeaders, backoff, buildInvestigateJsonRepairPrompt, @@ -493,6 +494,17 @@ export class ClaudeAgentSdkPlugin implements AgentPlugin { try { parsed = parseInvestigateResults(resultText, batch); } catch (err) { + if (err instanceof AgentPolicyRefusalError) { + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText, + error: err, + batch, + }); + throw err; + } yield { type: "thinking" as const, message: "Claude returned non-JSON investigation output; requesting JSON-only repair", @@ -727,6 +739,17 @@ export class ClaudeAgentSdkPlugin implements AgentPlugin { try { verdicts = parseRevalidateVerdicts(resultText); } catch (err) { + if (err instanceof AgentPolicyRefusalError) { + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText, + error: err, + batch, + }); + throw err; + } yield { type: "thinking" as const, message: "Claude returned non-JSON revalidation output; requesting JSON-only repair", diff --git a/packages/processor/src/agents/codex-sdk.ts b/packages/processor/src/agents/codex-sdk.ts index 9d3a778c..74d14cfb 100644 --- a/packages/processor/src/agents/codex-sdk.ts +++ b/packages/processor/src/agents/codex-sdk.ts @@ -13,6 +13,7 @@ import { type ThreadItem, } from "@openai/codex-sdk"; import { + AgentPolicyRefusalError, attributionHeaders, backoff, buildInvestigateJsonRepairPrompt, @@ -955,6 +956,17 @@ export class CodexAgentSdkPlugin implements AgentPlugin { try { parsedOutcome = parseInvestigateResults(resultText, batch); } catch (err) { + if (err instanceof AgentPolicyRefusalError) { + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText, + error: err, + batch, + }); + throw err; + } yield { type: "thinking" as const, message: "Codex returned non-JSON investigation output; requesting JSON-only repair", @@ -1240,6 +1252,17 @@ export class CodexAgentSdkPlugin implements AgentPlugin { try { verdicts = parseRevalidateVerdicts(resultText); } catch (err) { + if (err instanceof AgentPolicyRefusalError) { + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText, + error: err, + batch, + }); + throw err; + } yield { type: "thinking" as const, message: "Codex returned non-JSON revalidation output; requesting JSON-only repair", diff --git a/packages/processor/src/agents/pi-sdk.ts b/packages/processor/src/agents/pi-sdk.ts index fe9a0524..62cc6e8d 100644 --- a/packages/processor/src/agents/pi-sdk.ts +++ b/packages/processor/src/agents/pi-sdk.ts @@ -18,6 +18,7 @@ import { type ToolDefinition, } from "@earendil-works/pi-coding-agent"; import { + AgentPolicyRefusalError, attributionHeaders, backoff, buildInvestigateJsonRepairPrompt, @@ -965,6 +966,18 @@ export class PiAgentPlugin implements AgentPlugin { try { parsed = parseInvestigateResults(resultText, batch); } catch (err) { + if (err instanceof AgentPolicyRefusalError) { + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText, + error: err, + batch, + }); + session?.dispose(); + throw err; + } yield { type: "thinking", message: "Pi returned non-JSON investigation output; requesting JSON-only repair", @@ -1142,6 +1155,18 @@ export class PiAgentPlugin implements AgentPlugin { try { verdicts = parseRevalidateVerdicts(resultText); } catch (err) { + if (err instanceof AgentPolicyRefusalError) { + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText, + error: err, + batch, + }); + session?.dispose(); + throw err; + } yield { type: "thinking", message: "Pi returned non-JSON revalidation output; requesting JSON-only repair", diff --git a/packages/processor/src/agents/shared.ts b/packages/processor/src/agents/shared.ts index 6e3bdb3f..3c0dc584 100644 --- a/packages/processor/src/agents/shared.ts +++ b/packages/processor/src/agents/shared.ts @@ -218,6 +218,74 @@ export class QuotaExhaustedError extends Error { } } +// --- Policy refusal -------------------------------------------------------- + +/** + * Hard provider refusals (safety policy), distinct from quota: the provider + * declined the request itself, so the agent never produced findings JSON. + * - `anthropic-usage-policy` — Claude Code usage-policy (AUP) block + * - `openai-safety-system` — OpenAI `invalid_prompt` / safety-system prose + * - `content-policy` — generic `content_policy_violation` envelope + */ +type RefusalSource = "anthropic-usage-policy" | "openai-safety-system" | "content-policy"; + +/** + * Classify agent output as a hard provider policy refusal, or `undefined`. + * + * Matching is intentionally strict — the opposite trade-off from + * `classifyQuotaError`: a false positive here would relabel ordinary + * malformed JSON as "the model declined" (zero coverage over the batch), + * which is worse than the status quo. Misses fall through to the existing + * fail-loud parse error, which stays accurate. + */ +export function classifyPolicyRefusal(msg: string): RefusalSource | undefined { + if (!msg) return undefined; + const m = msg.toLowerCase(); + + // Claude Code usage-policy block: + // "API Error: Claude Code is unable to respond to this request, which + // appears to violate our Usage Policy (https://www.anthropic.com/legal/aup)." + if ( + /anthropic\.com\/legal\/aup/.test(m) || + (/unable to respond to this request/.test(m) && /usage polic/.test(m)) + ) { + return "anthropic-usage-policy"; + } + + // OpenAI safety-system rejection: the `invalid_prompt` error code, or the + // prose form that accompanies it. + if (/\binvalid_prompt\b/.test(m) || /rejected as a result of our safety system/.test(m)) { + return "openai-safety-system"; + } + + if (/\bcontent_policy_violation\b/.test(m)) { + return "content-policy"; + } + + return undefined; +} + +/** + * Thrown when the provider refused the request on policy grounds instead of + * producing findings JSON. Unlike `QuotaExhaustedError` this does NOT abort + * the run: a refusal depends on the batch's content, so other batches can + * proceed — but this batch must surface as errored, and must skip the + * JSON-repair follow-up (which would otherwise ask a declining model to + * re-emit conclusions it never produced, and accepts an empty findings + * array as a valid answer). + */ +export class AgentPolicyRefusalError extends Error { + readonly source: RefusalSource; + readonly rawMessage: string; + + constructor(source: RefusalSource, rawMessage: string) { + super(`Agent refused the request on policy grounds (${source}): ${rawMessage.slice(0, 300)}`); + this.name = "AgentPolicyRefusalError"; + this.source = source; + this.rawMessage = rawMessage; + } +} + /** * Vercel AI Gateway endpoints — duplicated from preflight.ts on purpose: * processor doesn't depend on the deepsec CLI package. Keep these in sync @@ -745,6 +813,14 @@ function parseAgentJsonArray(params: { try { parsed = JSON.parse(jsonPayload); } catch (strictErr) { + // Classify before attempting repair, so a refusal is never "repaired" + // into a valid-but-empty findings array. Doing it inside the catch — + // rather than before JSON.parse — means valid findings JSON that merely + // quotes refusal text as evidence parses normally. + const refusalSource = classifyPolicyRefusal(resultText); + if (refusalSource) { + throw new AgentPolicyRefusalError(refusalSource, resultText); + } const repairCandidate = extractArrayCandidateForRepair(jsonPayload); try { parsed = JSON.parse(jsonrepair(repairCandidate));