feat(studio): online evaluators

apps/agent-forge/e2e/a11y.spec.ts

@@ -128,8 +128,8 @@ async function expectNoSeriousAxeViolations(page: Page, label: string) {
 // Every route in the table (new screens are covered automatically): one <main>, one h1,
 // the banner and the primary nav, and an axe scan of the populated (stubbed daemon) state.
 // Plus the deep-link states that render a different view on the same route.
-// The datasets and queues routes scan the stub's dataset and annotation queue fixtures. Placeholder routes (/evals/:section, /evals/:section/:id) scan the Placeholder today; when their real
-// screens land, add their endpoints to e2e/observeStub.ts or they get scanned in a 404/empty state.
+// The datasets, queues and evaluators routes scan the stub's dataset, annotation queue and evaluator fixtures; a new
+// screen's endpoints go in e2e/observeStub.ts too, or it gets scanned in a 404/empty state.
 const SCANS = [
   ...ROUTES.map(({ pattern, example }) => ({ label: pattern, path: example })),
   ...['/traces?status=error', '/traces/e2e-trace?obs=e2e-obs', '/threads?status=active', '/prompts/e2e-prompt?diff=1,2'].map((path) => ({ label: path, path })),

apps/agent-forge/e2e/observeStub.ts

@@ -2,6 +2,7 @@ import type { Page, Route } from '@playwright/test';
 import type {
   ObserveAgentRow,
   ObserveDataset,
+  ObserveEvaluator,
   ObserveExample,
   ObserveHubEntry,
   ObserveInsights,
@@ -20,7 +21,8 @@ import type {
  * thread with one trace (an LLM call and a tool call), one prompt with two
  * versions, an eval's scores, a Context Hub namespace with a fleet and a
  * private entry, a dataset with two examples (one copied from the trace), and
- * an annotation queue whose next pending item is the trace.
+ * an annotation queue whose next pending item is the trace, and two online
+ * evaluator rules (a code rule with online scores, and a judge rule).
  */
 const NOW = Date.now();
 const T0 = NOW - 60_000;
@@ -225,6 +227,35 @@ const queueItem: ObserveQueueItem = {
   done_at: null,
 };
 
+const evaluators: ObserveEvaluator[] = [
+  {
+    rule_id: 'e2e-evaluator',
+    name: 'answered',
+    enabled: true,
+    filter: { agent_id: 'e2e-agent', status: 'ok' },
+    sample_rate: 1,
+    kind: 'code',
+    config: { source: "(trace) => (trace.output ? 1 : 0)" },
+    created_at: T0 - HOUR,
+    updated_at: T0 - HOUR,
+  },
+  {
+    rule_id: 'e2e-judge',
+    name: 'polite',
+    enabled: false,
+    filter: {},
+    sample_rate: 0.5,
+    kind: 'judge',
+    config: { model: 'openai/gpt-4o-mini', rubric: 'Is the reply polite?' },
+    created_at: T0 - HOUR,
+    updated_at: T0 - HOUR,
+  },
+];
+const onlineScores: ObserveScoreRow[] = [
+  { ...score, score_id: 'e2e-online-1', name: 'answered', source: 'online', value: 1, comment: null, obs_id: null },
+  { ...score, score_id: 'e2e-online-2', name: 'answered', source: 'online', value: 0, comment: null, obs_id: null, created_at: T0 - HOUR },
+];
+
 const json = (route: Route, body: unknown, status = 200) =>
   route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) });
 
@@ -260,9 +291,16 @@ export async function stubObserve(page: Page): Promise<void> {
       return json(route, { queues: [queue], total: 1, limit: Number(q.get('limit') ?? 100), offset: 0 });
     if (path === '/api/v1/queues/e2e-queue') return json(route, { queue });
     if (path === '/api/v1/queues/e2e-queue/next') return json(route, { item: queueItem });
+    if (path === '/api/v1/evaluators' && route.request().method() === 'GET') {
+      const rows = evaluators.filter((e) => (!q.get('kind') || e.kind === q.get('kind')) && (!q.get('enabled') || String(e.enabled) === q.get('enabled')));
+      return json(route, { evaluators: rows, total: rows.length, limit: Number(q.get('limit') ?? 100), offset: 0 });
+    }
+    if (path === '/api/v1/evaluators/e2e-evaluator' && route.request().method() === 'GET') return json(route, { evaluator: evaluators[0] });
+    if (path === '/api/v1/evaluators/e2e-evaluator/test') return json(route, { result: { value: 1, comment: '1 tool call(s)' } });
     if (path === '/api/v1/scores') {
       if (q.get('aggregate') === 'eval')
         return json(route, { evals: [{ name: 'e2e-eval', count: 4, passed: 3, pass_rate: 0.75, avg_value: 0.75, last_ts: T0 + 2_000 }] });
+      if (q.get('source') === 'online') return json(route, { scores: onlineScores });
       return json(route, { scores: [score, { ...score, score_id: 'e2e-score-2', value: 0, comment: 'wrong order id' }] });
     }
     return json(route, { error: 'not found' }, 404);

apps/agent-forge/e2e/revamp.spec.ts

@@ -127,7 +127,8 @@ const FILTERS: Record<ScreenId, string> = {
   datasets: '?offset=50',
   // Queues pages server-side the same way (`?offset=`).
   queues: '?offset=50',
-  evals: '',
+  // The list's kind/state filters (ridden along on the edit view for its way back) and the edit view's test-on-trace prefill.
+  evaluators: '?kind=code&enabled=true&trace=e2e-trace',
   prompts: '?diff=1,2',
   hub: '?agent=triage-bot&scope=fleet',
   settings: '',
@@ -200,6 +201,15 @@ const CONTROLS: Record<string, (main: Locator, page: Page) => Promise<void>> = {
   '/evals/queues': async (main) => {
     await expectPreviousPage(main);
   },
+  '/evals/evaluators': async (main) => {
+    await expect(main.getByRole('combobox', { name: 'Kind' })).toHaveValue('code');
+    await expect(main.getByRole('combobox', { name: 'State' })).toHaveValue('true');
+    await expect(main.getByRole('switch', { name: 'answered enabled' })).toBeChecked();
+  },
+  '/evals/evaluators/:id': async (main) => {
+    await expect(main.getByRole('textbox', { name: 'Trace id' })).toHaveValue('e2e-trace');
+    await expect(main.getByRole('link', { name: 'All evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true');
+  },
   '/prompts/:name': async (main) => {
     await expect(main.getByRole('combobox', { name: 'Compare version A' })).toHaveValue('1');
     await expect(main.getByRole('combobox', { name: 'Compare version B' })).toHaveValue('2');
@@ -221,8 +231,6 @@ CONTROLS['/hub/:namespace'] = CONTROLS['/hub'];
 const NO_CONTROLS = new Set([
   '/agents', // canonicalises to the open agent; nothing to filter
   '/threads/:sessionId', // a thread's turns; the list's filters only ride along in the URL
-  '/evals/:section', // evaluators / annotation placeholder screen
-  '/evals/:section/:id', // evaluators / annotation placeholder screen
   '/evals/queues/:id', // one queue's review pane; no URL-backed controls beyond the id
   '/prompts', // the registry list has no filters
   '/settings', // no URL state

apps/agent-forge/server/__tests__/app.test.ts

@@ -221,7 +221,12 @@ describe('settings/secrets HTTP routes', () => {
   let baseDir: string;
   let app: ReturnType<typeof createApp>;
 
+  // The provider env vars feed `envKey`: cleared for each test so a developer's own keys don't leak into the expectations.
+  const savedEnv = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY };
+
   beforeEach(() => {
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.ANTHROPIC_API_KEY;
     baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lou-r-app-test-'));
     const agentStore = createFsAgentStore(baseDir);
     const runManager = new RunManager({
@@ -235,22 +240,36 @@ describe('settings/secrets HTTP routes', () => {
   });
 
   afterEach(() => {
+    for (const [name, value] of Object.entries(savedEnv)) {
+      if (value === undefined) delete process.env[name];
+      else process.env[name] = value;
+    }
     fs.rmSync(baseDir, { recursive: true, force: true });
   });
 
   it('GET /settings/providers starts with no keys configured', async () => {
     const res = await request(app).get('/settings/providers');
     expect(res.status).toBe(200);
     expect(res.body).toEqual([
-      { provider: 'openai', hasKey: false, masked: null },
-      { provider: 'anthropic', hasKey: false, masked: null },
+      { provider: 'openai', hasKey: false, masked: null, envKey: false },
+      { provider: 'anthropic', hasKey: false, masked: null, envKey: false },
+    ]);
+  });
+
+  it('GET /settings/providers reports a provider whose env var is set as envKey, never its value', async () => {
+    process.env.OPENAI_API_KEY = 'env-key-placeholder';
+    const res = await request(app).get('/settings/providers');
+    expect(res.body).toEqual([
+      { provider: 'openai', hasKey: false, masked: null, envKey: true },
+      { provider: 'anthropic', hasKey: false, masked: null, envKey: false },
     ]);
+    expect(JSON.stringify(res.body)).not.toContain('env-key-placeholder');
   });
 
   it('PUT /settings/providers/:provider stores a key and never echoes it back', async () => {
     const res = await request(app).put('/settings/providers/openai').send({ apiKey: 'sk-super-secret' });
     expect(res.status).toBe(200);
-    expect(res.body).toEqual({ provider: 'openai', hasKey: true, masked: '••••••••cret' });
+    expect(res.body).toEqual({ provider: 'openai', hasKey: true, masked: '••••••••cret', envKey: false });
     expect(JSON.stringify(res.body)).not.toContain('sk-super-secret');
 
     const raw = fs.readFileSync(path.join(baseDir, '.lousho', 'secrets.json'), 'utf8');

apps/agent-forge/server/__tests__/observeProxy.test.ts

@@ -4,7 +4,8 @@
  * headers. A dead daemon answers 502 with the `lousho observe start` hint.
  * Writes go through only when OBSERVE_WRITE_ALLOWLIST names the method and
  * path (feedback scores, a prompt label move, a hub publish, dataset and
- * example writes, annotation queue writes); every other non-GET stays 405.
+ * example writes, annotation queue writes, evaluator rule writes); every
+ * other non-GET stays 405.
  */
 import { describe, it, expect, beforeEach, afterEach } from 'vitest';
 import * as http from 'node:http';
@@ -45,6 +46,12 @@ describe('observe proxy', () => {
       let body = '';
       req.on('data', (chunk: Buffer) => (body += chunk.toString()));
       req.on('end', () => {
+        // A code rule, so judge.ts's test route (registered before the proxy) forwards its /test here.
+        if (req.method === 'GET' && req.url === '/api/v1/evaluators/r1') {
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ evaluator: { rule_id: 'r1', kind: 'code' } }));
+          return;
+        }
         if (req.method === 'DELETE') {
           lastDelete = { url: req.url, contentType: req.headers['content-type'] ?? null, body };
           res.writeHead(204);
@@ -191,6 +198,46 @@ describe('observe proxy', () => {
     expect(res.status).toBe(405);
   });
 
+  it('forwards POST /api/v1/evaluators with its JSON body', async () => {
+    const body = { name: 'answered', kind: 'code', config: { source: '(trace) => 1' } };
+    const res = await request(app).post('/observe/api/v1/evaluators').send(body);
+    expect(res.status).toBe(201);
+    expect(res.body).toEqual({ echoed: '/api/v1/evaluators', method: 'POST', contentType: 'application/json', body });
+  });
+
+  it('forwards PUT /api/v1/evaluators/:id with its partial JSON body', async () => {
+    const res = await request(app).put('/observe/api/v1/evaluators/r1').send({ enabled: false });
+    expect(res.status).toBe(200);
+    expect(res.body).toEqual({ echoed: '/api/v1/evaluators/r1', method: 'PUT', contentType: 'application/json', body: { enabled: false } });
+  });
+
+  it('forwards DELETE /api/v1/evaluators/:id with no body and passes the 204 back', async () => {
+    lastDelete = undefined;
+    const res = await request(app).delete('/observe/api/v1/evaluators/r1');
+    expect(res.status).toBe(204);
+    expect(lastDelete).toEqual({ url: '/api/v1/evaluators/r1', contentType: null, body: '' });
+  });
+
+  it('leaves POST /api/v1/evaluators/:id/test to the Studio route, which forwards a code rule to the daemon', async () => {
+    const res = await request(app).post('/observe/api/v1/evaluators/r1/test').send({ trace_id: 't1' });
+    expect(res.status).toBe(201);
+    expect(res.body).toMatchObject({ echoed: '/api/v1/evaluators/r1/test', method: 'POST', body: { trace_id: 't1' } });
+  });
+
+  it.each([
+    ['delete', '/observe/api/v1/evaluators'],
+    ['put', '/observe/api/v1/evaluators'],
+    ['post', '/observe/api/v1/evaluators/r1'],
+    ['put', '/observe/api/v1/evaluators/r1/test'],
+    ['post', '/observe/api/v1/evaluators/judge-jobs/claim'],
+    ['post', '/observe/api/v1/evaluators/judge-jobs/j1/result'],
+    ['put', '/observe/api/v1/evaluators/%2e%2e'],
+    ['delete', '/observe/api/v1/evaluators/a%2Fb'],
+  ] as const)('keeps the evaluator-adjacent write %s %s at 405', async (method, url) => {
+    const res = await request(app)[method](url).send({});
+    expect(res.status).toBe(405);
+  });
+
   it.each([
     ['post', '/observe/api/v1/prompts'],
     ['put', '/observe/api/v1/prompts/mira'],
@@ -224,8 +271,10 @@ describe('observe proxy', () => {
     expect(res.status).toBe(413);
   });
 
-  it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes, dataset and queue writes', () => {
-    expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE', 'POST', 'POST', 'POST']);
+  it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes, dataset, queue and evaluator writes', () => {
+    expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual([
+      'POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE', 'POST', 'POST', 'POST', 'POST', 'PUT', 'DELETE',
+    ]);
     expect(OBSERVE_WRITE_ALLOWLIST.every((entry) => entry.path instanceof RegExp)).toBe(true);
   });
 });

apps/agent-forge/server/buildAgent.ts

@@ -31,7 +31,7 @@ import {
 } from '@lousho/build-ai-agent';
 import type { AgentFlow } from '@lousho/build-ai-agent/flows';
 import { compileHooksFromSpecPolicy } from './compileHooks';
-import { isSecretProvider, type SecretsStore } from './secretsStore';
+import { PROVIDER_KEY_ENV, isSecretProvider, type SecretsStore } from './secretsStore';
 
 /**
  * Local, server-only tools available to a spec's `tools` list in ADDITION
@@ -84,11 +84,6 @@ export interface BuiltAgent {
   sandbox: SandboxAdapter;
 }
 
-const REAL_KEYED_PROVIDER_ENV: Record<string, string> = {
-  openai: 'OPENAI_API_KEY',
-  anthropic: 'ANTHROPIC_API_KEY',
-};
-
 /** Real provider types this app doesn't manage a stored key for (no plain-API-key concept, or not yet wired into secretsStore) - env-var-only. */
 const REAL_ENV_ONLY_PROVIDERS = new Set(['ollama', 'openrouter']);
 
@@ -121,8 +116,8 @@ function resolveProviderForSpec(type: string, model: string, secretsStore?: Secr
 
   if (isSecretProvider(lower)) {
     const storedKey = secretsStore?.getKey(lower);
-    const key = storedKey || process.env[REAL_KEYED_PROVIDER_ENV[lower]];
-    if (!key) throw missingProviderKey(lower, REAL_KEYED_PROVIDER_ENV[lower]);
+    const key = storedKey || process.env[PROVIDER_KEY_ENV[lower]];
+    if (!key) throw missingProviderKey(lower, PROVIDER_KEY_ENV[lower]);
     return LLMProviderRegistry.create(lower, { defaultModel: model, apiKey: key });
   }
 

apps/agent-forge/server/observeProxy.ts

@@ -12,10 +12,11 @@
  * {@link OBSERVE_WRITE_ALLOWLIST} names the method and path - a short list of
  * Studio actions (human feedback scores, moving a prompt label, editing a
  * Context Hub fleet entry, dataset and example writes, annotation queue
- * writes) - with the JSON body
+ * writes, online evaluator rule create/update/delete) - with the JSON body
  * app.ts's `express.json` already parsed, re-serialised and capped at 1 MiB
  * (a DELETE forwards no body, and its 204 passes back empty). Every other non-GET - the ingest endpoint above all - answers 405
- * and stays reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it
+ * and stays reporter-only. (`POST /observe/api/v1/evaluators/:id/test` never
+ * reaches this proxy: app.ts registers judge.ts's route for it first.) `/observe` sits in app.ts's API_PREFIXES, so it
  * carries the same per-launch token requirement as the rest of the studio API.
  */
 import type { Express, Request } from 'express';
@@ -42,6 +43,9 @@ export const OBSERVE_WRITE_ALLOWLIST: readonly ObserveWrite[] = [
   { method: 'POST', path: /^\/api\/v1\/queues$/ },
   { method: 'POST', path: /^\/api\/v1\/queues\/[^/]+\/items$/ },
   { method: 'POST', path: /^\/api\/v1\/queues\/[^/]+\/items\/[^/]+\/complete$/ },
+  { method: 'POST', path: /^\/api\/v1\/evaluators$/ },
+  { method: 'PUT', path: /^\/api\/v1\/evaluators\/[^/]+$/ },
+  { method: 'DELETE', path: /^\/api\/v1\/evaluators\/[^/]+$/ },
 ];
 
 /**

apps/agent-forge/server/secretsStore.test.ts

@@ -8,20 +8,36 @@ describe('SecretsStore', () => {
   let baseDir: string;
   let store: SecretsStore;
 
+  const savedEnv = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY };
+
   beforeEach(() => {
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.ANTHROPIC_API_KEY;
     baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lou-r-secrets-'));
     store = new SecretsStore(baseDir);
   });
 
   afterEach(() => {
+    for (const [name, value] of Object.entries(savedEnv)) {
+      if (value === undefined) delete process.env[name];
+      else process.env[name] = value;
+    }
     fs.rmSync(baseDir, { recursive: true, force: true });
   });
 
   it('reports no key stored for a fresh provider', () => {
     const status = store.list();
     expect(status).toEqual([
-      { provider: 'openai', hasKey: false, masked: null },
-      { provider: 'anthropic', hasKey: false, masked: null },
+      { provider: 'openai', hasKey: false, masked: null, envKey: false },
+      { provider: 'anthropic', hasKey: false, masked: null, envKey: false },
+    ]);
+  });
+
+  it('reports envKey from the provider env var, independent of a stored key', () => {
+    process.env.ANTHROPIC_API_KEY = 'env-key-placeholder';
+    expect(store.list().map((s) => [s.provider, s.hasKey, s.envKey])).toEqual([
+      ['openai', false, false],
+      ['anthropic', false, true],
     ]);
   });
 

apps/agent-forge/server/secretsStore.ts

@@ -50,6 +50,12 @@ import type { ProviderKeyStatus } from '../shared/wireTypes';
 const SECRET_PROVIDERS = ['openai', 'anthropic'] as const;
 export type SecretProvider = (typeof SECRET_PROVIDERS)[number];
 
+/** The env var each managed provider falls back to when no key is stored (buildAgent.ts resolves runs with it; `list()` reports whether it is set). */
+export const PROVIDER_KEY_ENV: Record<SecretProvider, string> = {
+  openai: 'OPENAI_API_KEY',
+  anthropic: 'ANTHROPIC_API_KEY',
+};
+
 export function isSecretProvider(value: string): value is SecretProvider {
   return (SECRET_PROVIDERS as readonly string[]).includes(value);
 }
@@ -169,13 +175,13 @@ export class SecretsStore {
     }
   }
 
-  /** Masked status for every managed provider - safe to return from a GET route. */
+  /** Masked status for every managed provider, plus whether its env var is set (never its value) - safe to return from a GET route. */
   list(): ProviderKeyStatus[] {
     const records = this.loadRecords();
     return SECRET_PROVIDERS.map((provider) => {
       const hasKey = provider in records;
       const key = hasKey ? this.getKey(provider) : undefined;
-      return { provider, hasKey: hasKey && !!key, masked: key ? maskKey(key) : null };
+      return { provider, hasKey: hasKey && !!key, masked: key ? maskKey(key) : null, envKey: !!process.env[PROVIDER_KEY_ENV[provider]] };
     });
   }
 }

apps/agent-forge/shared/wireTypes.ts

@@ -216,6 +216,11 @@ export interface ProviderKeyStatus {
   hasKey: boolean;
   /** Last-4-visible masked form for display, e.g. '••••••••3f2a'. `null` when no key is stored. Never the real key. */
   masked: string | null;
+  /**
+   * The provider's env var (`OPENAI_API_KEY` / `ANTHROPIC_API_KEY`) is set in the server's environment. Runs fall back
+   * to it when no key is stored, so a provider with only `envKey` can still run. Never the value.
+   */
+  envKey: boolean;
 }
 
 /**