From 25dedbeff051cb4c7bca8566d10ad17be93a2e8b Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:22:09 +0300
Subject: [PATCH 1/8] LOU-G1: Implement defineEval() core API

Wraps vitest test() around the real static AgentExecutor.execute() call
(AgentExecutor has no instance methods in this codebase). EvalConfig's
execution fields (agent, input, provider, toolRegistry, maxSteps,
temperature, maxTokens) are typed directly off ExecuteOptions rather than
a nested {prompt, toolRegistry} shape, since that's what execute() actually
accepts. Manually verified score:()=>0/threshold:1 fails and score:()=>1
passes via a throwaway eval file targeted with an ad-hoc vitest config
(deleted before commit; LOU-G2 wires *.eval.ts into the default config).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/defineEval.ts | 73 +++++++++++++++++++++++++++++++++++++++++
 src/evals/index.ts      |  6 ++++
 src/index.ts            |  3 ++
 3 files changed, 82 insertions(+)
 create mode 100644 src/evals/defineEval.ts
 create mode 100644 src/evals/index.ts

diff --git a/src/evals/defineEval.ts b/src/evals/defineEval.ts
new file mode 100644
index 000000000..e8a445070
--- /dev/null
+++ b/src/evals/defineEval.ts
@@ -0,0 +1,73 @@
+/**
+ * defineEval() core API
+ *
+ * A thin wrapper around vitest's `test()` that runs a real agent through
+ * the REAL static AgentExecutor.execute() (this SDK's AgentExecutor has no
+ * instance methods - it is a static, instance-free API, see
+ * src/execution/AgentExecutor.ts), scores the resulting ExecutionResult
+ * with a caller-supplied `score()` function, and asserts the score meets
+ * a threshold.
+ */
+
+import { test, expect } from 'vitest';
+import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../execution/AgentExecutor';
+
+/**
+ * Configuration for a single eval case.
+ *
+ * The fields that map straight through to AgentExecutor.execute() are
+ * deliberately named/typed identically to ExecuteOptions (agent, input,
+ * provider, toolRegistry, maxSteps, temperature, maxTokens) so defineEval()
+ * can forward them without any reshaping - there is no nested `agent.prompt`
+ * / `agent.toolRegistry` shape here, because that is not what execute()
+ * actually accepts.
+ */
+export interface EvalConfig {
+  /** Test name, shown in vitest output. */
+  name: string;
+  /** Real AgentConfig, forwarded as-is to AgentExecutor.execute(). */
+  agent: ExecuteOptions['agent'];
+  /** Real input (string or Message[]), forwarded as-is. */
+  input: ExecuteOptions['input'];
+  /** Real LLMProvider instance, forwarded as-is. */
+  provider: ExecuteOptions['provider'];
+  /** Optional ToolRegistry, forwarded as-is. */
+  toolRegistry?: ExecuteOptions['toolRegistry'];
+  maxSteps?: ExecuteOptions['maxSteps'];
+  temperature?: ExecuteOptions['temperature'];
+  maxTokens?: ExecuteOptions['maxTokens'];
+  /**
+   * Scores the ExecutionResult from AgentExecutor.execute(). May be async
+   * (e.g. an llmJudge()-based scorer that itself calls out to a provider).
+   * Expected to return a number, conventionally in [0, 1] but the only
+   * hard requirement is that it be comparable to `threshold`.
+   */
+  score: (result: ExecutionResult) => number | Promise<number>;
+  /** Minimum score (inclusive) for the eval to pass. */
+  threshold: number;
+}
+
+/**
+ * Define a single eval as a vitest test. Calls AgentExecutor.execute()
+ * exactly once with the config's execution fields, scores the result, and
+ * asserts `score >= threshold`.
+ */
+export function defineEval(config: EvalConfig): void {
+  const { name, score, threshold, ...executeFields } = config;
+
+  test(name, async () => {
+    const result = await AgentExecutor.execute({
+      agent: executeFields.agent,
+      input: executeFields.input,
+      provider: executeFields.provider,
+      toolRegistry: executeFields.toolRegistry,
+      maxSteps: executeFields.maxSteps,
+      temperature: executeFields.temperature,
+      maxTokens: executeFields.maxTokens,
+    });
+
+    const resultScore = await score(result);
+
+    expect(resultScore).toBeGreaterThanOrEqual(threshold);
+  });
+}
diff --git a/src/evals/index.ts b/src/evals/index.ts
new file mode 100644
index 000000000..53cf03a46
--- /dev/null
+++ b/src/evals/index.ts
@@ -0,0 +1,6 @@
+/**
+ * Evals Module
+ * Lightweight harness for scoring agent runs (LOU-G)
+ */
+
+export * from './defineEval';
diff --git a/src/index.ts b/src/index.ts
index 835f94e47..6e294435d 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -32,6 +32,9 @@ export * from './providers';
 // Execution
 export * from './execution';
 
+// Evals
+export * from './evals';
+
 // Security
 export * from './security';
 

From 183ff66262d3293e3257d8504af499a6e6c72624 Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:23:18 +0300
Subject: [PATCH 2/8] LOU-G2: Wire *.eval.ts into vitest config

Adds an explicit test.include (vitest had no prior include, relying on its
default test/spec glob) covering both the default test/spec pattern and
'**/*.eval.ts', plus a '**/*.judge.eval.ts' exclude so LOU-G6's llmJudge
evals never run on the default `vitest run` path once they exist. Adds a
real, always-passing defineEval() smoke eval using MockLLMProvider,
confirmed picked up by `npx vitest run` and covered by `npm run
test:coverage`.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/__smoke__.eval.ts | 21 +++++++++++++++++++++
 vitest.config.ts            | 21 ++++++++++++++++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 src/evals/__smoke__.eval.ts

diff --git a/src/evals/__smoke__.eval.ts b/src/evals/__smoke__.eval.ts
new file mode 100644
index 000000000..6ac0dbcf8
--- /dev/null
+++ b/src/evals/__smoke__.eval.ts
@@ -0,0 +1,21 @@
+/**
+ * Smoke eval (LOU-G2) - a real, always-passing defineEval() call that
+ * proves '**\/*.eval.ts' files are picked up by the default `vitest run`
+ * path (wired into vitest.config.ts's test.include in this ticket).
+ */
+import { defineEval } from './defineEval';
+import { createMockProvider } from '../providers/mock';
+import { AgentType } from '../types';
+
+defineEval({
+  name: 'smoke: eval files are collected by default vitest run',
+  agent: {
+    name: 'Smoke Agent',
+    agentType: AgentType.SmartAssistant,
+    prompt: 'You are a helpful assistant',
+  },
+  input: 'Hello',
+  provider: createMockProvider({ name: 'mock', responses: ['Hi there!'] }),
+  score: (result) => (result.text.length > 0 ? 1 : 0),
+  threshold: 1,
+});
diff --git a/vitest.config.ts b/vitest.config.ts
index 345bdcff7..0fbe08b34 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -4,11 +4,30 @@ export default defineConfig({
   test: {
     globals: true,
     environment: 'node',
+    // Adds '**/*.eval.ts' (LOU-G2) alongside vitest's own default
+    // '**/*.{test,spec}.?(c|m)[jt]s?(x)' pattern (there was no explicit
+    // `include` before this, so the default test/spec pattern is spelled
+    // out here too) so eval files defined via defineEval() run as part of
+    // the normal `vitest run` / `npm test` path.
+    include: ['**/*.{test,spec}.?(c|m)[jt]s?(x)', '**/*.eval.ts'],
     // Excludes the LOU-E11 fixture repo's own test file - it's a plain
     // node:test suite meant to be run by the fixture's own `npm test`
     // (via createTestRunGuardrail() against a scratch copy of it), not
     // collected by vitest here.
-    exclude: ['**/node_modules/**', '**/dist/**', 'src/execution/__fixtures__/**'],
+    //
+    // Also excludes '**/*.judge.eval.ts' (LOU-G6): llmJudge()-based evals
+    // call out to a real LLM provider and must NEVER run in the default
+    // `vitest run` path. Without this exclude, '**/*.eval.ts' above would
+    // also match 'foo.judge.eval.ts' (it still ends in '.eval.ts'), so the
+    // exclude is required, not just the more specific include pattern.
+    // Judge evals are run exclusively via vitest.judge.config.ts /
+    // `npm run test:evals:judge`.
+    exclude: [
+      '**/node_modules/**',
+      '**/dist/**',
+      'src/execution/__fixtures__/**',
+      '**/*.judge.eval.ts',
+    ],
     coverage: {
       provider: 'v8',
       reporter: ['text', 'json', 'html'],

From a4041dc2b7ddc294b5bd946b8bc35203291d7447 Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:24:03 +0300
Subject: [PATCH 3/8] LOU-G3: Rule-based / exact-match scorer

Adds exactMatch(matcher) in src/evals/scorers.ts, matching a string,
RegExp, or predicate against the real ExecutionResult.text field. Never
throws for a non-match, including when a predicate itself throws. Unit
tested with one fixed fake ExecutionResult per matcher type, covering
match and non-match.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/index.ts        |  1 +
 src/evals/scorers.test.ts | 40 +++++++++++++++++++++++++++++++++
 src/evals/scorers.ts      | 47 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 88 insertions(+)
 create mode 100644 src/evals/scorers.test.ts
 create mode 100644 src/evals/scorers.ts

diff --git a/src/evals/index.ts b/src/evals/index.ts
index 53cf03a46..76b982099 100644
--- a/src/evals/index.ts
+++ b/src/evals/index.ts
@@ -4,3 +4,4 @@
  */
 
 export * from './defineEval';
+export * from './scorers';
diff --git a/src/evals/scorers.test.ts b/src/evals/scorers.test.ts
new file mode 100644
index 000000000..9d127e178
--- /dev/null
+++ b/src/evals/scorers.test.ts
@@ -0,0 +1,40 @@
+import { describe, it, expect } from 'vitest';
+import { exactMatch } from './scorers';
+import { ExecutionResult } from '../execution/AgentExecutor';
+
+function fakeResult(text: string): ExecutionResult {
+  return {
+    text,
+    messages: [],
+    toolCalls: [],
+    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+    finishReason: 'stop',
+    steps: 1,
+  };
+}
+
+describe('exactMatch', () => {
+  it('scores 1 on exact string match, 0 on mismatch', () => {
+    const scorer = exactMatch('hello world');
+    expect(scorer(fakeResult('hello world'))).toBe(1);
+    expect(scorer(fakeResult('goodbye world'))).toBe(0);
+  });
+
+  it('scores 1 when a RegExp matches, 0 when it does not', () => {
+    const scorer = exactMatch(/^\d+ items$/);
+    expect(scorer(fakeResult('42 items'))).toBe(1);
+    expect(scorer(fakeResult('forty-two items'))).toBe(0);
+  });
+
+  it('scores 1/0 based on a predicate function, without throwing for a predicate that throws', () => {
+    const scorer = exactMatch((text: string) => text.includes('success'));
+    expect(scorer(fakeResult('operation success'))).toBe(1);
+    expect(scorer(fakeResult('operation failed'))).toBe(0);
+
+    const throwingScorer = exactMatch(() => {
+      throw new Error('predicate blew up');
+    });
+    expect(() => throwingScorer(fakeResult('anything'))).not.toThrow();
+    expect(throwingScorer(fakeResult('anything'))).toBe(0);
+  });
+});
diff --git a/src/evals/scorers.ts b/src/evals/scorers.ts
new file mode 100644
index 000000000..23d6a03bd
--- /dev/null
+++ b/src/evals/scorers.ts
@@ -0,0 +1,47 @@
+/**
+ * Scorers for defineEval() (LOU-G3/G4/G5)
+ *
+ * All scorer factories return a plain, synchronous `(result: ExecutionResult) => number`
+ * function - the same shape defineEval()'s `score` field expects. None of
+ * them throw on a non-match/mismatch; they simply return 0.
+ */
+
+import { ExecutionResult } from '../execution/AgentExecutor';
+
+// ---------------------------------------------------------------------------
+// LOU-G3: rule-based / exact-match scorer
+// ---------------------------------------------------------------------------
+
+/**
+ * A matcher against the ExecutionResult's `.text` field: exact string
+ * equality, a RegExp tested against the text, or a predicate function.
+ */
+export type Matcher = string | RegExp | ((text: string) => boolean);
+
+/**
+ * Returns a scorer that checks `result.text` against `matcher`:
+ *  - string: exact equality
+ *  - RegExp: `.test(text)`
+ *  - function: called with `text`, its boolean return is used
+ *
+ * Never throws for a non-match - always resolves to 1 (match) or 0
+ * (no match).
+ */
+export function exactMatch(matcher: Matcher): (result: ExecutionResult) => number {
+  return (result: ExecutionResult): number => {
+    const text = result.text ?? '';
+
+    try {
+      if (typeof matcher === 'string') {
+        return text === matcher ? 1 : 0;
+      }
+      if (matcher instanceof RegExp) {
+        return matcher.test(text) ? 1 : 0;
+      }
+      return matcher(text) ? 1 : 0;
+    } catch {
+      // A throwing predicate is treated as a non-match, not a scorer crash.
+      return 0;
+    }
+  };
+}

From f1af57f86d2a1f808f7d69a696fba33149e998d0 Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:24:37 +0300
Subject: [PATCH 4/8] LOU-G4: Tool-call-order assertion scorer

Adds toolCallOrder(expected) in src/evals/scorers.ts. AgentExecutor's real
ExecutionResult already exposes a top-level toolCalls: ToolCall[] field
(populated across every loop step in runAgentLoop(), in request order), so
this scorer reads it directly rather than collecting calls itself via the
LOU-E onToolCall/onToolResult tracing hooks. Compares length, per-position
tool name, and (when specified) args via JSON.stringify equality against
the parsed function.arguments. Unit tests cover exact match, shuffled
order, a missing call, an args mismatch, and omitted args.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/index.ts        |  1 +
 src/evals/scorers.test.ts | 54 ++++++++++++++++++++++++++++++++--
 src/evals/scorers.ts      | 61 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 113 insertions(+), 3 deletions(-)

diff --git a/src/evals/index.ts b/src/evals/index.ts
index 76b982099..3879f52f0 100644
--- a/src/evals/index.ts
+++ b/src/evals/index.ts
@@ -5,3 +5,4 @@
 
 export * from './defineEval';
 export * from './scorers';
+
diff --git a/src/evals/scorers.test.ts b/src/evals/scorers.test.ts
index 9d127e178..f4a67c518 100644
--- a/src/evals/scorers.test.ts
+++ b/src/evals/scorers.test.ts
@@ -1,18 +1,27 @@
 import { describe, it, expect } from 'vitest';
-import { exactMatch } from './scorers';
+import { exactMatch, toolCallOrder } from './scorers';
 import { ExecutionResult } from '../execution/AgentExecutor';
+import { ToolCall } from '../providers/llm';
 
-function fakeResult(text: string): ExecutionResult {
+function fakeResult(text: string, toolCalls: ToolCall[] = []): ExecutionResult {
   return {
     text,
     messages: [],
-    toolCalls: [],
+    toolCalls,
     usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
     finishReason: 'stop',
     steps: 1,
   };
 }
 
+function fakeToolCall(id: string, name: string, args: Record<string, unknown>): ToolCall {
+  return {
+    id,
+    type: 'function',
+    function: { name, arguments: JSON.stringify(args) },
+  };
+}
+
 describe('exactMatch', () => {
   it('scores 1 on exact string match, 0 on mismatch', () => {
     const scorer = exactMatch('hello world');
@@ -38,3 +47,42 @@ describe('exactMatch', () => {
     expect(throwingScorer(fakeResult('anything'))).toBe(0);
   });
 });
+
+describe('toolCallOrder', () => {
+  const searchCall = fakeToolCall('call_1', 'search', { query: 'cats' });
+  const summarizeCall = fakeToolCall('call_2', 'summarize', { maxWords: 50 });
+
+  it('scores 1 for an exact-order, exact-args match', () => {
+    const scorer = toolCallOrder([
+      { tool: 'search', args: { query: 'cats' } },
+      { tool: 'summarize', args: { maxWords: 50 } },
+    ]);
+    expect(scorer(fakeResult('done', [searchCall, summarizeCall]))).toBe(1);
+  });
+
+  it('scores 0 for a shuffled order', () => {
+    const scorer = toolCallOrder([
+      { tool: 'search', args: { query: 'cats' } },
+      { tool: 'summarize', args: { maxWords: 50 } },
+    ]);
+    expect(scorer(fakeResult('done', [summarizeCall, searchCall]))).toBe(0);
+  });
+
+  it('scores 0 for a missing call (length mismatch)', () => {
+    const scorer = toolCallOrder([
+      { tool: 'search', args: { query: 'cats' } },
+      { tool: 'summarize', args: { maxWords: 50 } },
+    ]);
+    expect(scorer(fakeResult('done', [searchCall]))).toBe(0);
+  });
+
+  it('scores 0 on args mismatch even when tool names and order match', () => {
+    const scorer = toolCallOrder([{ tool: 'search', args: { query: 'dogs' } }]);
+    expect(scorer(fakeResult('done', [searchCall]))).toBe(0);
+  });
+
+  it('ignores args entirely when not specified in the expected call', () => {
+    const scorer = toolCallOrder([{ tool: 'search' }]);
+    expect(scorer(fakeResult('done', [searchCall]))).toBe(1);
+  });
+});
diff --git a/src/evals/scorers.ts b/src/evals/scorers.ts
index 23d6a03bd..605d0a4de 100644
--- a/src/evals/scorers.ts
+++ b/src/evals/scorers.ts
@@ -7,6 +7,7 @@
  */
 
 import { ExecutionResult } from '../execution/AgentExecutor';
+import { ToolCall } from '../providers/llm';
 
 // ---------------------------------------------------------------------------
 // LOU-G3: rule-based / exact-match scorer
@@ -45,3 +46,63 @@ export function exactMatch(matcher: Matcher): (result: ExecutionResult) => numbe
     }
   };
 }
+
+// ---------------------------------------------------------------------------
+// LOU-G4: tool-call-order assertion scorer
+// ---------------------------------------------------------------------------
+
+/**
+ * A single expected tool call in an expected sequence. `args`, when given,
+ * is compared to the actual parsed call arguments via JSON.stringify
+ * equality.
+ */
+export interface ExpectedCall {
+  tool: string;
+  args?: Record<string, unknown>;
+}
+
+/**
+ * How tool calls actually reach the scorer: AgentExecutor.execute()'s real
+ * ExecutionResult (src/execution/AgentExecutor.ts) already carries a
+ * top-level `toolCalls: ToolCall[]` field. It's populated as the run's
+ * loop accumulates every tool call the model made
+ * (`allToolCalls.push(...result.toolCalls)` in runAgentLoop()), across
+ * every step, in request order. There is no need to collect calls via the
+ * LOU-E onToolCall/onToolResult tracing hooks for this - the plain field
+ * on the result already exposes the full, ordered call list. Each entry's
+ * `function.arguments` is a JSON string (the raw provider tool-call
+ * arguments), so args comparison below JSON.parses it before comparing.
+ */
+export function toolCallOrder(expected: ExpectedCall[]): (result: ExecutionResult) => number {
+  return (result: ExecutionResult): number => {
+    const actual: ToolCall[] = result.toolCalls ?? [];
+
+    if (actual.length !== expected.length) {
+      return 0;
+    }
+
+    for (let i = 0; i < expected.length; i++) {
+      const expectedCall = expected[i];
+      const actualCall = actual[i];
+
+      if (actualCall.function.name !== expectedCall.tool) {
+        return 0;
+      }
+
+      if (expectedCall.args !== undefined) {
+        let actualArgs: unknown;
+        try {
+          actualArgs = JSON.parse(actualCall.function.arguments);
+        } catch {
+          return 0;
+        }
+
+        if (JSON.stringify(actualArgs) !== JSON.stringify(expectedCall.args)) {
+          return 0;
+        }
+      }
+    }
+
+    return 1;
+  };
+}

From da2b3050563dbfaba64e8f8e840557f41cecf287 Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:25:03 +0300
Subject: [PATCH 5/8] LOU-G5: Token/step budget scorer

Adds budget(limits) and describeBudgetFailure(result, limits) in
src/evals/scorers.ts, checking the real result.usage.totalTokens and
result.steps fields against optional maxTokens/maxSteps limits. budget()
returns 0 if either exceeded, 1 otherwise; describeBudgetFailure() checks
both and reports a clause per failure. Unit tests cover a result at 50% of
budget (pass), 150% (fail), and assert the failure message contains both
the actual and budgeted numbers.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/scorers.test.ts | 48 +++++++++++++++++++++++++++++++---
 src/evals/scorers.ts      | 54 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 98 insertions(+), 4 deletions(-)

diff --git a/src/evals/scorers.test.ts b/src/evals/scorers.test.ts
index f4a67c518..7d453a6e6 100644
--- a/src/evals/scorers.test.ts
+++ b/src/evals/scorers.test.ts
@@ -1,16 +1,20 @@
 import { describe, it, expect } from 'vitest';
-import { exactMatch, toolCallOrder } from './scorers';
+import { exactMatch, toolCallOrder, budget, describeBudgetFailure } from './scorers';
 import { ExecutionResult } from '../execution/AgentExecutor';
 import { ToolCall } from '../providers/llm';
 
-function fakeResult(text: string, toolCalls: ToolCall[] = []): ExecutionResult {
+function fakeResult(
+  text: string,
+  toolCalls: ToolCall[] = [],
+  overrides: Partial<Pick<ExecutionResult, 'usage' | 'steps'>> = {}
+): ExecutionResult {
   return {
     text,
     messages: [],
     toolCalls,
-    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+    usage: overrides.usage ?? { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
     finishReason: 'stop',
-    steps: 1,
+    steps: overrides.steps ?? 1,
   };
 }
 
@@ -86,3 +90,39 @@ describe('toolCallOrder', () => {
     expect(scorer(fakeResult('done', [searchCall]))).toBe(1);
   });
 });
+
+describe('budget', () => {
+  it('scores 1 at 50% of the budget', () => {
+    const scorer = budget({ maxTokens: 1000, maxSteps: 10 });
+    const result = fakeResult('done', [], {
+      usage: { promptTokens: 300, completionTokens: 200, totalTokens: 500 },
+      steps: 5,
+    });
+    expect(scorer(result)).toBe(1);
+  });
+
+  it('scores 0 at 150% of the budget', () => {
+    const scorer = budget({ maxTokens: 1000, maxSteps: 10 });
+    const result = fakeResult('done', [], {
+      usage: { promptTokens: 900, completionTokens: 600, totalTokens: 1500 },
+      steps: 15,
+    });
+    expect(scorer(result)).toBe(0);
+  });
+});
+
+describe('describeBudgetFailure', () => {
+  it('includes both the actual and budgeted numbers as substrings', () => {
+    const limits = { maxTokens: 1000, maxSteps: 10 };
+    const result = fakeResult('done', [], {
+      usage: { promptTokens: 900, completionTokens: 600, totalTokens: 1500 },
+      steps: 15,
+    });
+    const message = describeBudgetFailure(result, limits);
+
+    expect(message).toContain('1500');
+    expect(message).toContain('1000');
+    expect(message).toContain('15');
+    expect(message).toContain('10');
+  });
+});
diff --git a/src/evals/scorers.ts b/src/evals/scorers.ts
index 605d0a4de..e73427c6c 100644
--- a/src/evals/scorers.ts
+++ b/src/evals/scorers.ts
@@ -106,3 +106,57 @@ export function toolCallOrder(expected: ExpectedCall[]): (result: ExecutionResul
     return 1;
   };
 }
+
+// ---------------------------------------------------------------------------
+// LOU-G5: token/step budget scorer
+// ---------------------------------------------------------------------------
+
+export interface Budget {
+  maxTokens?: number;
+  maxSteps?: number;
+}
+
+/**
+ * Returns a scorer that checks `result.usage.totalTokens` and `result.steps`
+ * (the real field names on ExecutionResult) against `limits`. Returns 0 if
+ * either configured limit is exceeded, 1 otherwise. A limit left
+ * `undefined` is not checked.
+ */
+export function budget(limits: Budget): (result: ExecutionResult) => number {
+  return (result: ExecutionResult): number => {
+    if (limits.maxTokens !== undefined && result.usage.totalTokens > limits.maxTokens) {
+      return 0;
+    }
+    if (limits.maxSteps !== undefined && result.steps > limits.maxSteps) {
+      return 0;
+    }
+    return 1;
+  };
+}
+
+/**
+ * Produces a human-readable description of which budget(s) a result
+ * exceeded, e.g. "tokens 1500 exceeded maxTokens 1000 by 500". Checks both
+ * tokens and steps, and includes a clause for each that failed.
+ */
+export function describeBudgetFailure(result: ExecutionResult, limits: Budget): string {
+  const failures: string[] = [];
+
+  if (limits.maxTokens !== undefined && result.usage.totalTokens > limits.maxTokens) {
+    const over = result.usage.totalTokens - limits.maxTokens;
+    failures.push(
+      `tokens ${result.usage.totalTokens} exceeded maxTokens ${limits.maxTokens} by ${over}`
+    );
+  }
+
+  if (limits.maxSteps !== undefined && result.steps > limits.maxSteps) {
+    const over = result.steps - limits.maxSteps;
+    failures.push(`steps ${result.steps} exceeded maxSteps ${limits.maxSteps} by ${over}`);
+  }
+
+  if (failures.length === 0) {
+    return 'no budget exceeded';
+  }
+
+  return failures.join('; ');
+}

From da66ab611810a9c49d4166d0f55321f2171fd39e Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:26:36 +0300
Subject: [PATCH 6/8] LOU-G6: llmJudge() rubric grading helper

Adds src/evals/llmJudge.ts using the REAL LLMProvider interface
(src/providers/llm.ts): it has generate()/stream() taking a GenerateOptions
with a `messages` array, no bare-string `.complete()`, so llmJudge() builds
a grading prompt and calls provider.generate({ model, messages: [{role:
'user', content: gradePrompt}] }). parseJudgeScore() clamps a well-formed
numeric response to [0, 1] and explicitly returns { score: 0, reason } for
a malformed/non-numeric response, verified by a dedicated unit test rather
than relying on NaN's accidental fail-closed comparison behavior.

Adds a standalone vitest.judge.config.ts (NOT merged with the base config)
whose include matches ONLY '**/*.judge.eval.ts'; vitest.config.ts's own
'**/*.eval.ts' include (LOU-G2) already excludes that same suffix. Added
"test:evals:judge" npm script; left .github/workflows/ci.yml untouched.

Manually verified (not committed): bumping the smoke judge eval's threshold
above the mock judge's fixed 0.9 score flips it from pass to fail, proving
defineEval()+llmJudge() wiring tracks the fixed score exactly. Confirmed
`npx vitest run` (default config) does not collect *.judge.eval.ts, and
neither vitest.config.ts's include nor ci.yml reference judge evals.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 package.json                      |  1 +
 src/evals/__smoke__.judge.eval.ts | 29 ++++++++++++
 src/evals/index.ts                |  1 +
 src/evals/llmJudge.test.ts        | 34 ++++++++++++++
 src/evals/llmJudge.ts             | 78 +++++++++++++++++++++++++++++++
 vitest.judge.config.ts            | 20 ++++++++
 6 files changed, 163 insertions(+)
 create mode 100644 src/evals/__smoke__.judge.eval.ts
 create mode 100644 src/evals/llmJudge.test.ts
 create mode 100644 src/evals/llmJudge.ts
 create mode 100644 vitest.judge.config.ts

diff --git a/package.json b/package.json
index 82b3bea6f..876cae16c 100644
--- a/package.json
+++ b/package.json
@@ -50,6 +50,7 @@
     "dev": "tsup --watch",
     "test": "vitest",
     "test:coverage": "vitest run --coverage --config vitest.coverage.config.ts",
+    "test:evals:judge": "vitest run --config vitest.judge.config.ts",
     "lint": "eslint src",
     "typecheck": "tsc --noEmit",
     "example:tracing:console": "tsx examples/tracing/run-console.ts",
diff --git a/src/evals/__smoke__.judge.eval.ts b/src/evals/__smoke__.judge.eval.ts
new file mode 100644
index 000000000..142000aa8
--- /dev/null
+++ b/src/evals/__smoke__.judge.eval.ts
@@ -0,0 +1,29 @@
+/**
+ * Smoke judge eval (LOU-G6) - a real defineEval() + llmJudge() combination,
+ * using a mock judge provider that resolves generate() with a fixed
+ * rubric score ("0.9"). Only collected by vitest.judge.config.ts's
+ * '**\/*.judge.eval.ts' include (NOT by the default vitest.config.ts,
+ * whose '**\/*.eval.ts' include explicitly excludes this filename), so run
+ * this file via `npm run test:evals:judge`.
+ */
+import { defineEval } from './defineEval';
+import { llmJudge } from './llmJudge';
+import { createMockProvider } from '../providers/mock';
+import { AgentType } from '../types';
+
+defineEval({
+  name: 'smoke (judge): llmJudge scores a fixed judge response',
+  agent: {
+    name: 'Smoke Agent',
+    agentType: AgentType.SmartAssistant,
+    prompt: 'You are a helpful assistant',
+  },
+  input: 'Summarize the weather',
+  provider: createMockProvider({ name: 'mock', responses: ['It is sunny today.'] }),
+  score: llmJudge({
+    provider: createMockProvider({ name: 'mock-judge', responses: ['0.9'] }),
+    model: 'mock-model',
+    rubric: 'Score 1 if the output mentions the weather, 0 otherwise.',
+  }),
+  threshold: 0.8,
+});
diff --git a/src/evals/index.ts b/src/evals/index.ts
index 3879f52f0..205597707 100644
--- a/src/evals/index.ts
+++ b/src/evals/index.ts
@@ -5,4 +5,5 @@
 
 export * from './defineEval';
 export * from './scorers';
+export * from './llmJudge';
 
diff --git a/src/evals/llmJudge.test.ts b/src/evals/llmJudge.test.ts
new file mode 100644
index 000000000..18cafd93f
--- /dev/null
+++ b/src/evals/llmJudge.test.ts
@@ -0,0 +1,34 @@
+import { describe, it, expect } from 'vitest';
+import { parseJudgeScore } from './llmJudge';
+
+describe('parseJudgeScore', () => {
+  it('parses a well-formed numeric response', () => {
+    expect(parseJudgeScore('0.9')).toEqual({ score: 0.9 });
+    expect(parseJudgeScore('1')).toEqual({ score: 1 });
+    expect(parseJudgeScore('  0.5  ')).toEqual({ score: 0.5 });
+  });
+
+  it('clamps out-of-range numeric responses into [0, 1]', () => {
+    expect(parseJudgeScore('1.5').score).toBe(1);
+    expect(parseJudgeScore('-0.3').score).toBe(0);
+  });
+
+  it('returns 0 with a clear reason for a malformed, non-numeric response, instead of letting NaN through', () => {
+    const { score, reason } = parseJudgeScore('the output looks great');
+
+    // Explicitly verify this is a real 0, not a NaN that happens to fail
+    // `>= threshold` comparisons (NaN comparisons are always false in JS,
+    // which would already "fail safe" - but that's accidental, not
+    // intentional, so we assert the actual numeric value here).
+    expect(score).toBe(0);
+    expect(Number.isNaN(score)).toBe(false);
+    expect(reason).toBeDefined();
+    expect(reason).toContain('the output looks great');
+  });
+
+  it('treats an empty response the same as a malformed one', () => {
+    const { score, reason } = parseJudgeScore('');
+    expect(score).toBe(0);
+    expect(reason).toBeDefined();
+  });
+});
diff --git a/src/evals/llmJudge.ts b/src/evals/llmJudge.ts
new file mode 100644
index 000000000..bdb3524c7
--- /dev/null
+++ b/src/evals/llmJudge.ts
@@ -0,0 +1,78 @@
+/**
+ * llmJudge() rubric grading helper (LOU-G6)
+ *
+ * Builds a grading prompt from a rubric and the agent run's output text,
+ * sends it through the REAL LLMProvider interface's generate() method
+ * (src/providers/llm.ts - LLMProvider only has generate()/stream(), both
+ * taking a GenerateOptions with a `messages` array; there is no
+ * `.complete(prompt: string)` method), and parses the response into a
+ * clamped [0, 1] score.
+ */
+
+import { LLMProvider } from '../providers/llm';
+import { ExecutionResult } from '../execution/AgentExecutor';
+
+export interface LLMJudgeConfig {
+  /** Real LLMProvider instance used to grade the output. */
+  provider: LLMProvider;
+  /** Model id passed through to provider.generate(). */
+  model: string;
+  /** Grading rubric / instructions shown to the judge model. */
+  rubric: string;
+  temperature?: number;
+}
+
+/**
+ * Builds the grading prompt sent to the judge model.
+ */
+function buildGradePrompt(rubric: string, outputText: string): string {
+  return [
+    'You are grading the output of an AI agent against a rubric.',
+    'Respond with ONLY a single number between 0 and 1 (inclusive), representing how well the output satisfies the rubric.',
+    'Do not include any explanation, units, or extra text - just the number.',
+    '',
+    `Rubric:\n${rubric}`,
+    '',
+    `Agent output to grade:\n${outputText}`,
+  ].join('\n');
+}
+
+/**
+ * Parses a judge model's raw text response into a score clamped to [0, 1].
+ * A malformed (non-numeric) response is handled explicitly - it returns 0
+ * with a reason, rather than letting `parseFloat` produce NaN and relying
+ * on NaN's comparisons always being false in `expect(score).toBeGreaterThanOrEqual(threshold)`
+ * to accidentally fail closed.
+ */
+export function parseJudgeScore(rawText: string): { score: number; reason?: string } {
+  const trimmed = rawText.trim();
+  const parsed = parseFloat(trimmed);
+
+  if (Number.isNaN(parsed)) {
+    return { score: 0, reason: `judge response was not a number: ${JSON.stringify(rawText)}` };
+  }
+
+  const clamped = Math.min(1, Math.max(0, parsed));
+  return { score: clamped };
+}
+
+/**
+ * Returns a defineEval()-compatible scorer that grades an ExecutionResult's
+ * `.text` against `config.rubric` using an LLM judge, via
+ * provider.generate({ messages: [...] }) (NOT a bare-string `.complete()`,
+ * which this codebase's LLMProvider interface does not have).
+ */
+export function llmJudge(config: LLMJudgeConfig): (result: ExecutionResult) => Promise<number> {
+  return async (result: ExecutionResult): Promise<number> => {
+    const gradePrompt = buildGradePrompt(config.rubric, result.text ?? '');
+
+    const response = await config.provider.generate({
+      model: config.model,
+      messages: [{ role: 'user', content: gradePrompt }],
+      temperature: config.temperature,
+    });
+
+    const { score } = parseJudgeScore(response.text);
+    return score;
+  };
+}
diff --git a/vitest.judge.config.ts b/vitest.judge.config.ts
new file mode 100644
index 000000000..fcad27766
--- /dev/null
+++ b/vitest.judge.config.ts
@@ -0,0 +1,20 @@
+import { defineConfig } from 'vitest/config';
+
+// Standalone config (NOT merged with vitest.config.ts) for LOU-G6 judge
+// evals: llmJudge()-based evals call out to a real LLM provider and must
+// NEVER run as part of the default `vitest run` / `npm run test:coverage`
+// path. This config's `include` matches ONLY '**/*.judge.eval.ts', and the
+// main vitest.config.ts (LOU-G2) explicitly excludes that same pattern
+// from its own '**/*.eval.ts' include, so the two configs' file sets never
+// overlap.
+//
+// Run explicitly via `npm run test:evals:judge`
+// (`vitest run --config vitest.judge.config.ts`).
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['**/*.judge.eval.ts'],
+    exclude: ['**/node_modules/**', '**/dist/**'],
+  },
+});

From d2b2a71cf90ac40d47e42fca57bcddc9f8878d5f Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:31:03 +0300
Subject: [PATCH 7/8] LOU-G fix: add structural runtime guard preventing
 llmJudge() from running outside its judge-eval config
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

QA found that llmJudge()'s exclusion from default CI relied purely on the
'*.judge.eval.ts' file-naming convention and glob patterns — a misnamed or
mis-globbed judge-eval file (wrong case, different suffix) could silently
make a real, budgeted LLM call as part of the default `vitest run`/CI path,
with no structural check to catch it.

llmJudge() now throws unless LOUSHY_ALLOW_LLM_JUDGE=1 is set, which only
vitest.judge.config.ts sets. This is independent of how the test file is
named, so the isolation boundary holds even if a file is accidentally
picked up by the wrong vitest config.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/llmJudge.test.ts | 38 ++++++++++++++++++++++++++++++++++++--
 src/evals/llmJudge.ts      | 13 +++++++++++++
 vitest.judge.config.ts     |  5 +++++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/src/evals/llmJudge.test.ts b/src/evals/llmJudge.test.ts
index 18cafd93f..22a5d44cf 100644
--- a/src/evals/llmJudge.test.ts
+++ b/src/evals/llmJudge.test.ts
@@ -1,5 +1,6 @@
-import { describe, it, expect } from 'vitest';
-import { parseJudgeScore } from './llmJudge';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { parseJudgeScore, llmJudge } from './llmJudge';
+import type { LLMProvider } from '../providers/llm';
 
 describe('parseJudgeScore', () => {
   it('parses a well-formed numeric response', () => {
@@ -32,3 +33,36 @@ describe('parseJudgeScore', () => {
     expect(reason).toBeDefined();
   });
 });
+
+describe('llmJudge() structural isolation guard', () => {
+  const originalEnv = process.env.LOUSHY_ALLOW_LLM_JUDGE;
+
+  beforeEach(() => {
+    delete process.env.LOUSHY_ALLOW_LLM_JUDGE;
+  });
+
+  afterEach(() => {
+    if (originalEnv === undefined) {
+      delete process.env.LOUSHY_ALLOW_LLM_JUDGE;
+    } else {
+      process.env.LOUSHY_ALLOW_LLM_JUDGE = originalEnv;
+    }
+  });
+
+  const mockProvider: LLMProvider = {
+    generate: async () => ({ text: '0.9' }) as never,
+  } as never;
+
+  it('refuses to run outside the judge-eval runner (LOUSHY_ALLOW_LLM_JUDGE unset)', async () => {
+    const scorer = llmJudge({ provider: mockProvider, model: 'test-model', rubric: 'be good' });
+    await expect(scorer({ text: 'hello' } as never)).rejects.toThrow(
+      /llmJudge\(\) was invoked outside the judge-eval runner/
+    );
+  });
+
+  it('runs normally when LOUSHY_ALLOW_LLM_JUDGE=1 (set by vitest.judge.config.ts)', async () => {
+    process.env.LOUSHY_ALLOW_LLM_JUDGE = '1';
+    const scorer = llmJudge({ provider: mockProvider, model: 'test-model', rubric: 'be good' });
+    await expect(scorer({ text: 'hello' } as never)).resolves.toBe(0.9);
+  });
+});
diff --git a/src/evals/llmJudge.ts b/src/evals/llmJudge.ts
index bdb3524c7..8b54741a9 100644
--- a/src/evals/llmJudge.ts
+++ b/src/evals/llmJudge.ts
@@ -64,6 +64,19 @@ export function parseJudgeScore(rawText: string): { score: number; reason?: stri
  */
 export function llmJudge(config: LLMJudgeConfig): (result: ExecutionResult) => Promise<number> {
   return async (result: ExecutionResult): Promise<number> => {
+    // Structural guard, independent of file-naming conventions: this env var
+    // is only set by vitest.judge.config.ts. A *.eval.ts file that imports
+    // llmJudge() but is accidentally picked up by the main vitest run (e.g.
+    // a misnamed or mis-globbed file) fails loudly here instead of silently
+    // making a real, budgeted LLM call as part of default/CI test runs.
+    if (process.env.LOUSHY_ALLOW_LLM_JUDGE !== '1') {
+      throw new Error(
+        'llmJudge() was invoked outside the judge-eval runner. ' +
+          'llmJudge()-based evals must live in a "*.judge.eval.ts" file and run via ' +
+          '`npm run test:evals:judge` (vitest.judge.config.ts), never the default `vitest run`.'
+      );
+    }
+
     const gradePrompt = buildGradePrompt(config.rubric, result.text ?? '');
 
     const response = await config.provider.generate({
diff --git a/vitest.judge.config.ts b/vitest.judge.config.ts
index fcad27766..aa9ff4782 100644
--- a/vitest.judge.config.ts
+++ b/vitest.judge.config.ts
@@ -16,5 +16,10 @@ export default defineConfig({
     environment: 'node',
     include: ['**/*.judge.eval.ts'],
     exclude: ['**/node_modules/**', '**/dist/**'],
+    // Structural (not just naming-convention) enforcement: llmJudge() itself
+    // refuses to run unless this env var is set, so a misnamed/mis-globbed
+    // judge-eval file picked up by the main config can't silently make a
+    // real LLM call in default CI - see src/evals/llmJudge.ts.
+    env: { LOUSHY_ALLOW_LLM_JUDGE: '1' },
   },
 });

From 6ea75db9cd0800c3f94cd8ccd7919b0d6b318a77 Mon Sep 17 00:00:00 2001
From: Ali Mohammad <freelancealimohammad@gmail.com>
Date: Mon, 28 Sep 2026 21:34:49 +0300
Subject: [PATCH 8/8] LOU-G fix: add integration test verifying toolCallOrder()
 against genuine AgentExecutor output

LOU-G4's AC requires toolCallOrder() to be checked against real
AgentExecutor.execute() output, not only hand-fabricated fakeResult()/
fakeToolCall() fixtures that could silently drift from how the real
executor actually populates toolCalls (field names, arguments-as-JSON-
string, ordering). Add a test that registers two real tools on a real
ToolRegistry, drives a scripted LLMProvider (same pattern as
AgentExecutor.test.ts) through a real AgentExecutor.execute() call, and
feeds the genuine result.toolCalls into toolCallOrder() - asserting 1
for the correct sequence and 0 for a reversed/incomplete one.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/evals/scorers.test.ts | 140 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 138 insertions(+), 2 deletions(-)

diff --git a/src/evals/scorers.test.ts b/src/evals/scorers.test.ts
index 7d453a6e6..1abf1ab38 100644
--- a/src/evals/scorers.test.ts
+++ b/src/evals/scorers.test.ts
@@ -1,7 +1,10 @@
 import { describe, it, expect } from 'vitest';
 import { exactMatch, toolCallOrder, budget, describeBudgetFailure } from './scorers';
-import { ExecutionResult } from '../execution/AgentExecutor';
-import { ToolCall } from '../providers/llm';
+import { AgentExecutor, ExecutionResult } from '../execution/AgentExecutor';
+import { ToolCall, LLMProvider, GenerateOptions, GenerateResult } from '../providers/llm';
+import { ToolRegistry } from '../tools';
+import { AgentBuilder } from '../core';
+import { AgentType } from '../types';
 
 function fakeResult(
   text: string,
@@ -111,6 +114,139 @@ describe('budget', () => {
   });
 });
 
+describe('toolCallOrder() against a genuine AgentExecutor.execute() result', () => {
+  // LOU-G4's acceptance criteria requires toolCallOrder() to be verified
+  // against REAL AgentExecutor.execute() output, not just fakeResult()/
+  // fakeToolCall() fixtures - a scorer can type-check against the
+  // ExecutionResult/ToolCall shapes while silently drifting from how the
+  // real executor actually populates `toolCalls` (field naming, whether
+  // `arguments` is a JSON string vs. object, ordering guarantees, etc.).
+  // This runs a real agent loop with two real tools registered in a real
+  // ToolRegistry, driven by a scripted LLMProvider (the same
+  // "scripted provider" pattern AgentExecutor.test.ts uses for multi-step
+  // tool-calling tests), and feeds the genuine result.toolCalls into
+  // toolCallOrder().
+
+  it('scores 1 for the correct order and 0 for a wrong order, on a real execute() result', async () => {
+    const toolRegistry = new ToolRegistry();
+
+    const searchExecute = async (args: Record<string, unknown>) => ({ found: args.query });
+    const summarizeExecute = async (args: Record<string, unknown>) => ({
+      summary: `summary of ${args.maxWords} words`,
+    });
+
+    toolRegistry.register('search', {
+      displayName: 'Search',
+      tool: {
+        description: 'Searches for something',
+        parameters: {},
+        execute: searchExecute,
+      } as any,
+    });
+    toolRegistry.register('summarize', {
+      displayName: 'Summarize',
+      tool: {
+        description: 'Summarizes something',
+        parameters: {},
+        execute: summarizeExecute,
+      } as any,
+    });
+
+    const agent = AgentBuilder.create()
+      .setType(AgentType.SmartAssistant)
+      .setName('Test Agent')
+      .addTool('search', { tool: 'search', options: {} })
+      .addTool('summarize', { tool: 'summarize', options: {} })
+      .build();
+
+    // Scripted provider: on step 1 requests `search`, on step 2 requests
+    // `summarize`, then stops - mirroring the scripted-provider pattern
+    // used by AgentExecutor.test.ts (a plain object implementing
+    // LLMProvider with a generate() that counts calls and returns
+    // different ToolCall[] per step).
+    let call = 0;
+    const scriptedProvider: LLMProvider = {
+      name: 'scripted',
+      supportsTools: () => true,
+      supportsStreaming: () => false,
+      getModels: async () => ['scripted'],
+      stream: async () => {
+        throw new Error('not implemented');
+      },
+      generate: async (_options: GenerateOptions): Promise<GenerateResult> => {
+        call++;
+        if (call === 1) {
+          return {
+            text: '',
+            finishReason: 'tool_calls',
+            usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+            toolCalls: [
+              {
+                id: 'call-1',
+                type: 'function',
+                function: { name: 'search', arguments: JSON.stringify({ query: 'cats' }) },
+              },
+            ],
+          };
+        }
+        if (call === 2) {
+          return {
+            text: '',
+            finishReason: 'tool_calls',
+            usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+            toolCalls: [
+              {
+                id: 'call-2',
+                type: 'function',
+                function: { name: 'summarize', arguments: JSON.stringify({ maxWords: 50 }) },
+              },
+            ],
+          };
+        }
+        return {
+          text: 'done',
+          finishReason: 'stop',
+          usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+        };
+      },
+    };
+
+    // A genuine AgentExecutor.execute() call - not a stub, not a mock of
+    // execute() itself. Only the LLMProvider is scripted; the executor's
+    // own tool-calling loop, message building and toolCalls accumulation
+    // all run for real.
+    const result = await AgentExecutor.execute({
+      agent,
+      input: 'search then summarize',
+      provider: scriptedProvider,
+      toolRegistry,
+    });
+
+    // Sanity: the real executor really did call both tools, in order, and
+    // really did populate `arguments` as a JSON string (not an object) -
+    // exactly the drift risk this test exists to catch.
+    expect(result.toolCalls).toHaveLength(2);
+    expect(result.toolCalls[0].function.name).toBe('search');
+    expect(result.toolCalls[1].function.name).toBe('summarize');
+    expect(typeof result.toolCalls[0].function.arguments).toBe('string');
+
+    const correctScorer = toolCallOrder([
+      { tool: 'search', args: { query: 'cats' } },
+      { tool: 'summarize', args: { maxWords: 50 } },
+    ]);
+    expect(correctScorer(result)).toBe(1);
+
+    const reversedScorer = toolCallOrder([
+      { tool: 'summarize', args: { maxWords: 50 } },
+      { tool: 'search', args: { query: 'cats' } },
+    ]);
+    expect(reversedScorer(result)).toBe(0);
+
+    const missingCallScorer = toolCallOrder([{ tool: 'search', args: { query: 'cats' } }]);
+    expect(missingCallScorer(result)).toBe(0);
+  });
+});
+
 describe('describeBudgetFailure', () => {
   it('includes both the actual and budgeted numbers as substrings', () => {
     const limits = { maxTokens: 1000, maxSteps: 10 };