LOU-G: Evals harness

package.json

@@ -50,6 +50,7 @@
     "dev": "tsup --watch",
     "test": "vitest",
     "test:coverage": "vitest run --coverage --config vitest.coverage.config.ts",
+    "test:evals:judge": "vitest run --config vitest.judge.config.ts",
     "lint": "eslint src",
     "typecheck": "tsc --noEmit",
     "example:tracing:console": "tsx examples/tracing/run-console.ts",

src/evals/__smoke__.eval.ts

@@ -0,0 +1,21 @@
+/**
+ * Smoke eval (LOU-G2) - a real, always-passing defineEval() call that
+ * proves '**\/*.eval.ts' files are picked up by the default `vitest run`
+ * path (wired into vitest.config.ts's test.include in this ticket).
+ */
+import { defineEval } from './defineEval';
+import { createMockProvider } from '../providers/mock';
+import { AgentType } from '../types';
+
+defineEval({
+  name: 'smoke: eval files are collected by default vitest run',
+  agent: {
+    name: 'Smoke Agent',
+    agentType: AgentType.SmartAssistant,
+    prompt: 'You are a helpful assistant',
+  },
+  input: 'Hello',
+  provider: createMockProvider({ name: 'mock', responses: ['Hi there!'] }),
+  score: (result) => (result.text.length > 0 ? 1 : 0),
+  threshold: 1,
+});

src/evals/__smoke__.judge.eval.ts

@@ -0,0 +1,29 @@
+/**
+ * Smoke judge eval (LOU-G6) - a real defineEval() + llmJudge() combination,
+ * using a mock judge provider that resolves generate() with a fixed
+ * rubric score ("0.9"). Only collected by vitest.judge.config.ts's
+ * '**\/*.judge.eval.ts' include (NOT by the default vitest.config.ts,
+ * whose '**\/*.eval.ts' include explicitly excludes this filename), so run
+ * this file via `npm run test:evals:judge`.
+ */
+import { defineEval } from './defineEval';
+import { llmJudge } from './llmJudge';
+import { createMockProvider } from '../providers/mock';
+import { AgentType } from '../types';
+
+defineEval({
+  name: 'smoke (judge): llmJudge scores a fixed judge response',
+  agent: {
+    name: 'Smoke Agent',
+    agentType: AgentType.SmartAssistant,
+    prompt: 'You are a helpful assistant',
+  },
+  input: 'Summarize the weather',
+  provider: createMockProvider({ name: 'mock', responses: ['It is sunny today.'] }),
+  score: llmJudge({
+    provider: createMockProvider({ name: 'mock-judge', responses: ['0.9'] }),
+    model: 'mock-model',
+    rubric: 'Score 1 if the output mentions the weather, 0 otherwise.',
+  }),
+  threshold: 0.8,
+});

src/evals/defineEval.ts

@@ -0,0 +1,73 @@
+/**
+ * defineEval() core API
+ *
+ * A thin wrapper around vitest's `test()` that runs a real agent through
+ * the REAL static AgentExecutor.execute() (this SDK's AgentExecutor has no
+ * instance methods - it is a static, instance-free API, see
+ * src/execution/AgentExecutor.ts), scores the resulting ExecutionResult
+ * with a caller-supplied `score()` function, and asserts the score meets
+ * a threshold.
+ */
+
+import { test, expect } from 'vitest';
+import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../execution/AgentExecutor';
+
+/**
+ * Configuration for a single eval case.
+ *
+ * The fields that map straight through to AgentExecutor.execute() are
+ * deliberately named/typed identically to ExecuteOptions (agent, input,
+ * provider, toolRegistry, maxSteps, temperature, maxTokens) so defineEval()
+ * can forward them without any reshaping - there is no nested `agent.prompt`
+ * / `agent.toolRegistry` shape here, because that is not what execute()
+ * actually accepts.
+ */
+export interface EvalConfig {
+  /** Test name, shown in vitest output. */
+  name: string;
+  /** Real AgentConfig, forwarded as-is to AgentExecutor.execute(). */
+  agent: ExecuteOptions['agent'];
+  /** Real input (string or Message[]), forwarded as-is. */
+  input: ExecuteOptions['input'];
+  /** Real LLMProvider instance, forwarded as-is. */
+  provider: ExecuteOptions['provider'];
+  /** Optional ToolRegistry, forwarded as-is. */
+  toolRegistry?: ExecuteOptions['toolRegistry'];
+  maxSteps?: ExecuteOptions['maxSteps'];
+  temperature?: ExecuteOptions['temperature'];
+  maxTokens?: ExecuteOptions['maxTokens'];
+  /**
+   * Scores the ExecutionResult from AgentExecutor.execute(). May be async
+   * (e.g. an llmJudge()-based scorer that itself calls out to a provider).
+   * Expected to return a number, conventionally in [0, 1] but the only
+   * hard requirement is that it be comparable to `threshold`.
+   */
+  score: (result: ExecutionResult) => number | Promise<number>;
+  /** Minimum score (inclusive) for the eval to pass. */
+  threshold: number;
+}
+
+/**
+ * Define a single eval as a vitest test. Calls AgentExecutor.execute()
+ * exactly once with the config's execution fields, scores the result, and
+ * asserts `score >= threshold`.
+ */
+export function defineEval(config: EvalConfig): void {
+  const { name, score, threshold, ...executeFields } = config;
+
+  test(name, async () => {
+    const result = await AgentExecutor.execute({
+      agent: executeFields.agent,
+      input: executeFields.input,
+      provider: executeFields.provider,
+      toolRegistry: executeFields.toolRegistry,
+      maxSteps: executeFields.maxSteps,
+      temperature: executeFields.temperature,
+      maxTokens: executeFields.maxTokens,
+    });
+
+    const resultScore = await score(result);
+
+    expect(resultScore).toBeGreaterThanOrEqual(threshold);
+  });
+}

src/evals/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Evals Module
+ * Lightweight harness for scoring agent runs (LOU-G)
+ */
+
+export * from './defineEval';
+export * from './scorers';
+export * from './llmJudge';
+

src/evals/llmJudge.test.ts

@@ -0,0 +1,68 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { parseJudgeScore, llmJudge } from './llmJudge';
+import type { LLMProvider } from '../providers/llm';
+
+describe('parseJudgeScore', () => {
+  it('parses a well-formed numeric response', () => {
+    expect(parseJudgeScore('0.9')).toEqual({ score: 0.9 });
+    expect(parseJudgeScore('1')).toEqual({ score: 1 });
+    expect(parseJudgeScore('  0.5  ')).toEqual({ score: 0.5 });
+  });
+
+  it('clamps out-of-range numeric responses into [0, 1]', () => {
+    expect(parseJudgeScore('1.5').score).toBe(1);
+    expect(parseJudgeScore('-0.3').score).toBe(0);
+  });
+
+  it('returns 0 with a clear reason for a malformed, non-numeric response, instead of letting NaN through', () => {
+    const { score, reason } = parseJudgeScore('the output looks great');
+
+    // Explicitly verify this is a real 0, not a NaN that happens to fail
+    // `>= threshold` comparisons (NaN comparisons are always false in JS,
+    // which would already "fail safe" - but that's accidental, not
+    // intentional, so we assert the actual numeric value here).
+    expect(score).toBe(0);
+    expect(Number.isNaN(score)).toBe(false);
+    expect(reason).toBeDefined();
+    expect(reason).toContain('the output looks great');
+  });
+
+  it('treats an empty response the same as a malformed one', () => {
+    const { score, reason } = parseJudgeScore('');
+    expect(score).toBe(0);
+    expect(reason).toBeDefined();
+  });
+});
+
+describe('llmJudge() structural isolation guard', () => {
+  const originalEnv = process.env.LOUSHY_ALLOW_LLM_JUDGE;
+
+  beforeEach(() => {
+    delete process.env.LOUSHY_ALLOW_LLM_JUDGE;
+  });
+
+  afterEach(() => {
+    if (originalEnv === undefined) {
+      delete process.env.LOUSHY_ALLOW_LLM_JUDGE;
+    } else {
+      process.env.LOUSHY_ALLOW_LLM_JUDGE = originalEnv;
+    }
+  });
+
+  const mockProvider: LLMProvider = {
+    generate: async () => ({ text: '0.9' }) as never,
+  } as never;
+
+  it('refuses to run outside the judge-eval runner (LOUSHY_ALLOW_LLM_JUDGE unset)', async () => {
+    const scorer = llmJudge({ provider: mockProvider, model: 'test-model', rubric: 'be good' });
+    await expect(scorer({ text: 'hello' } as never)).rejects.toThrow(
+      /llmJudge\(\) was invoked outside the judge-eval runner/
+    );
+  });
+
+  it('runs normally when LOUSHY_ALLOW_LLM_JUDGE=1 (set by vitest.judge.config.ts)', async () => {
+    process.env.LOUSHY_ALLOW_LLM_JUDGE = '1';
+    const scorer = llmJudge({ provider: mockProvider, model: 'test-model', rubric: 'be good' });
+    await expect(scorer({ text: 'hello' } as never)).resolves.toBe(0.9);
+  });
+});

src/evals/llmJudge.ts

@@ -0,0 +1,91 @@
+/**
+ * llmJudge() rubric grading helper (LOU-G6)
+ *
+ * Builds a grading prompt from a rubric and the agent run's output text,
+ * sends it through the REAL LLMProvider interface's generate() method
+ * (src/providers/llm.ts - LLMProvider only has generate()/stream(), both
+ * taking a GenerateOptions with a `messages` array; there is no
+ * `.complete(prompt: string)` method), and parses the response into a
+ * clamped [0, 1] score.
+ */
+
+import { LLMProvider } from '../providers/llm';
+import { ExecutionResult } from '../execution/AgentExecutor';
+
+export interface LLMJudgeConfig {
+  /** Real LLMProvider instance used to grade the output. */
+  provider: LLMProvider;
+  /** Model id passed through to provider.generate(). */
+  model: string;
+  /** Grading rubric / instructions shown to the judge model. */
+  rubric: string;
+  temperature?: number;
+}
+
+/**
+ * Builds the grading prompt sent to the judge model.
+ */
+function buildGradePrompt(rubric: string, outputText: string): string {
+  return [
+    'You are grading the output of an AI agent against a rubric.',
+    'Respond with ONLY a single number between 0 and 1 (inclusive), representing how well the output satisfies the rubric.',
+    'Do not include any explanation, units, or extra text - just the number.',
+    '',
+    `Rubric:\n${rubric}`,
+    '',
+    `Agent output to grade:\n${outputText}`,
+  ].join('\n');
+}
+
+/**
+ * Parses a judge model's raw text response into a score clamped to [0, 1].
+ * A malformed (non-numeric) response is handled explicitly - it returns 0
+ * with a reason, rather than letting `parseFloat` produce NaN and relying
+ * on NaN's comparisons always being false in `expect(score).toBeGreaterThanOrEqual(threshold)`
+ * to accidentally fail closed.
+ */
+export function parseJudgeScore(rawText: string): { score: number; reason?: string } {
+  const trimmed = rawText.trim();
+  const parsed = parseFloat(trimmed);
+
+  if (Number.isNaN(parsed)) {
+    return { score: 0, reason: `judge response was not a number: ${JSON.stringify(rawText)}` };
+  }
+
+  const clamped = Math.min(1, Math.max(0, parsed));
+  return { score: clamped };
+}
+
+/**
+ * Returns a defineEval()-compatible scorer that grades an ExecutionResult's
+ * `.text` against `config.rubric` using an LLM judge, via
+ * provider.generate({ messages: [...] }) (NOT a bare-string `.complete()`,
+ * which this codebase's LLMProvider interface does not have).
+ */
+export function llmJudge(config: LLMJudgeConfig): (result: ExecutionResult) => Promise<number> {
+  return async (result: ExecutionResult): Promise<number> => {
+    // Structural guard, independent of file-naming conventions: this env var
+    // is only set by vitest.judge.config.ts. A *.eval.ts file that imports
+    // llmJudge() but is accidentally picked up by the main vitest run (e.g.
+    // a misnamed or mis-globbed file) fails loudly here instead of silently
+    // making a real, budgeted LLM call as part of default/CI test runs.
+    if (process.env.LOUSHY_ALLOW_LLM_JUDGE !== '1') {
+      throw new Error(
+        'llmJudge() was invoked outside the judge-eval runner. ' +
+          'llmJudge()-based evals must live in a "*.judge.eval.ts" file and run via ' +
+          '`npm run test:evals:judge` (vitest.judge.config.ts), never the default `vitest run`.'
+      );
+    }
+
+    const gradePrompt = buildGradePrompt(config.rubric, result.text ?? '');
+
+    const response = await config.provider.generate({
+      model: config.model,
+      messages: [{ role: 'user', content: gradePrompt }],
+      temperature: config.temperature,
+    });
+
+    const { score } = parseJudgeScore(response.text);
+    return score;
+  };
+}