diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index b04f2498..668eb7a7 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -128,8 +128,8 @@ async function expectNoSeriousAxeViolations(page: Page, label: string) { // Every route in the table (new screens are covered automatically): one
, one h1, // the banner and the primary nav, and an axe scan of the populated (stubbed daemon) state. // Plus the deep-link states that render a different view on the same route. -// The datasets and queues routes scan the stub's dataset and annotation queue fixtures. Placeholder routes (/evals/:section, /evals/:section/:id) scan the Placeholder today; when their real -// screens land, add their endpoints to e2e/observeStub.ts or they get scanned in a 404/empty state. +// The datasets, queues and evaluators routes scan the stub's dataset, annotation queue and evaluator fixtures; a new +// screen's endpoints go in e2e/observeStub.ts too, or it gets scanned in a 404/empty state. const SCANS = [ ...ROUTES.map(({ pattern, example }) => ({ label: pattern, path: example })), ...['/traces?status=error', '/traces/e2e-trace?obs=e2e-obs', '/threads?status=active', '/prompts/e2e-prompt?diff=1,2'].map((path) => ({ label: path, path })), diff --git a/apps/agent-forge/e2e/observeStub.ts b/apps/agent-forge/e2e/observeStub.ts index 8c6f8b7d..67bd0ed6 100644 --- a/apps/agent-forge/e2e/observeStub.ts +++ b/apps/agent-forge/e2e/observeStub.ts @@ -2,6 +2,7 @@ import type { Page, Route } from '@playwright/test'; import type { ObserveAgentRow, ObserveDataset, + ObserveEvaluator, ObserveExample, ObserveHubEntry, ObserveInsights, @@ -20,7 +21,8 @@ import type { * thread with one trace (an LLM call and a tool call), one prompt with two * versions, an eval's scores, a Context Hub namespace with a fleet and a * private entry, a dataset with two examples (one copied from the trace), and - * an annotation queue whose next pending item is the trace. + * an annotation queue whose next pending item is the trace, and two online + * evaluator rules (a code rule with online scores, and a judge rule). */ const NOW = Date.now(); const T0 = NOW - 60_000; @@ -225,6 +227,35 @@ const queueItem: ObserveQueueItem = { done_at: null, }; +const evaluators: ObserveEvaluator[] = [ + { + rule_id: 'e2e-evaluator', + name: 'answered', + enabled: true, + filter: { agent_id: 'e2e-agent', status: 'ok' }, + sample_rate: 1, + kind: 'code', + config: { source: "(trace) => (trace.output ? 1 : 0)" }, + created_at: T0 - HOUR, + updated_at: T0 - HOUR, + }, + { + rule_id: 'e2e-judge', + name: 'polite', + enabled: false, + filter: {}, + sample_rate: 0.5, + kind: 'judge', + config: { model: 'openai/gpt-4o-mini', rubric: 'Is the reply polite?' }, + created_at: T0 - HOUR, + updated_at: T0 - HOUR, + }, +]; +const onlineScores: ObserveScoreRow[] = [ + { ...score, score_id: 'e2e-online-1', name: 'answered', source: 'online', value: 1, comment: null, obs_id: null }, + { ...score, score_id: 'e2e-online-2', name: 'answered', source: 'online', value: 0, comment: null, obs_id: null, created_at: T0 - HOUR }, +]; + const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); @@ -260,9 +291,16 @@ export async function stubObserve(page: Page): Promise { return json(route, { queues: [queue], total: 1, limit: Number(q.get('limit') ?? 100), offset: 0 }); if (path === '/api/v1/queues/e2e-queue') return json(route, { queue }); if (path === '/api/v1/queues/e2e-queue/next') return json(route, { item: queueItem }); + if (path === '/api/v1/evaluators' && route.request().method() === 'GET') { + const rows = evaluators.filter((e) => (!q.get('kind') || e.kind === q.get('kind')) && (!q.get('enabled') || String(e.enabled) === q.get('enabled'))); + return json(route, { evaluators: rows, total: rows.length, limit: Number(q.get('limit') ?? 100), offset: 0 }); + } + if (path === '/api/v1/evaluators/e2e-evaluator' && route.request().method() === 'GET') return json(route, { evaluator: evaluators[0] }); + if (path === '/api/v1/evaluators/e2e-evaluator/test') return json(route, { result: { value: 1, comment: '1 tool call(s)' } }); if (path === '/api/v1/scores') { if (q.get('aggregate') === 'eval') return json(route, { evals: [{ name: 'e2e-eval', count: 4, passed: 3, pass_rate: 0.75, avg_value: 0.75, last_ts: T0 + 2_000 }] }); + if (q.get('source') === 'online') return json(route, { scores: onlineScores }); return json(route, { scores: [score, { ...score, score_id: 'e2e-score-2', value: 0, comment: 'wrong order id' }] }); } return json(route, { error: 'not found' }, 404); diff --git a/apps/agent-forge/e2e/revamp.spec.ts b/apps/agent-forge/e2e/revamp.spec.ts index c8a6da1b..19a0cbe3 100644 --- a/apps/agent-forge/e2e/revamp.spec.ts +++ b/apps/agent-forge/e2e/revamp.spec.ts @@ -127,7 +127,8 @@ const FILTERS: Record = { datasets: '?offset=50', // Queues pages server-side the same way (`?offset=`). queues: '?offset=50', - evals: '', + // The list's kind/state filters (ridden along on the edit view for its way back) and the edit view's test-on-trace prefill. + evaluators: '?kind=code&enabled=true&trace=e2e-trace', prompts: '?diff=1,2', hub: '?agent=triage-bot&scope=fleet', settings: '', @@ -200,6 +201,15 @@ const CONTROLS: Record Promise> = { '/evals/queues': async (main) => { await expectPreviousPage(main); }, + '/evals/evaluators': async (main) => { + await expect(main.getByRole('combobox', { name: 'Kind' })).toHaveValue('code'); + await expect(main.getByRole('combobox', { name: 'State' })).toHaveValue('true'); + await expect(main.getByRole('switch', { name: 'answered enabled' })).toBeChecked(); + }, + '/evals/evaluators/:id': async (main) => { + await expect(main.getByRole('textbox', { name: 'Trace id' })).toHaveValue('e2e-trace'); + await expect(main.getByRole('link', { name: 'All evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true'); + }, '/prompts/:name': async (main) => { await expect(main.getByRole('combobox', { name: 'Compare version A' })).toHaveValue('1'); await expect(main.getByRole('combobox', { name: 'Compare version B' })).toHaveValue('2'); @@ -221,8 +231,6 @@ CONTROLS['/hub/:namespace'] = CONTROLS['/hub']; const NO_CONTROLS = new Set([ '/agents', // canonicalises to the open agent; nothing to filter '/threads/:sessionId', // a thread's turns; the list's filters only ride along in the URL - '/evals/:section', // evaluators / annotation placeholder screen - '/evals/:section/:id', // evaluators / annotation placeholder screen '/evals/queues/:id', // one queue's review pane; no URL-backed controls beyond the id '/prompts', // the registry list has no filters '/settings', // no URL state diff --git a/apps/agent-forge/server/__tests__/app.test.ts b/apps/agent-forge/server/__tests__/app.test.ts index 9e444a0f..f5a3fe43 100644 --- a/apps/agent-forge/server/__tests__/app.test.ts +++ b/apps/agent-forge/server/__tests__/app.test.ts @@ -221,7 +221,12 @@ describe('settings/secrets HTTP routes', () => { let baseDir: string; let app: ReturnType; + // The provider env vars feed `envKey`: cleared for each test so a developer's own keys don't leak into the expectations. + const savedEnv = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY }; + beforeEach(() => { + delete process.env.OPENAI_API_KEY; + delete process.env.ANTHROPIC_API_KEY; baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lou-r-app-test-')); const agentStore = createFsAgentStore(baseDir); const runManager = new RunManager({ @@ -235,6 +240,10 @@ describe('settings/secrets HTTP routes', () => { }); afterEach(() => { + for (const [name, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } fs.rmSync(baseDir, { recursive: true, force: true }); }); @@ -242,15 +251,25 @@ describe('settings/secrets HTTP routes', () => { const res = await request(app).get('/settings/providers'); expect(res.status).toBe(200); expect(res.body).toEqual([ - { provider: 'openai', hasKey: false, masked: null }, - { provider: 'anthropic', hasKey: false, masked: null }, + { provider: 'openai', hasKey: false, masked: null, envKey: false }, + { provider: 'anthropic', hasKey: false, masked: null, envKey: false }, + ]); + }); + + it('GET /settings/providers reports a provider whose env var is set as envKey, never its value', async () => { + process.env.OPENAI_API_KEY = 'env-key-placeholder'; + const res = await request(app).get('/settings/providers'); + expect(res.body).toEqual([ + { provider: 'openai', hasKey: false, masked: null, envKey: true }, + { provider: 'anthropic', hasKey: false, masked: null, envKey: false }, ]); + expect(JSON.stringify(res.body)).not.toContain('env-key-placeholder'); }); it('PUT /settings/providers/:provider stores a key and never echoes it back', async () => { const res = await request(app).put('/settings/providers/openai').send({ apiKey: 'sk-super-secret' }); expect(res.status).toBe(200); - expect(res.body).toEqual({ provider: 'openai', hasKey: true, masked: '••••••••cret' }); + expect(res.body).toEqual({ provider: 'openai', hasKey: true, masked: '••••••••cret', envKey: false }); expect(JSON.stringify(res.body)).not.toContain('sk-super-secret'); const raw = fs.readFileSync(path.join(baseDir, '.lousho', 'secrets.json'), 'utf8'); diff --git a/apps/agent-forge/server/__tests__/observeProxy.test.ts b/apps/agent-forge/server/__tests__/observeProxy.test.ts index 95713b13..ed400077 100644 --- a/apps/agent-forge/server/__tests__/observeProxy.test.ts +++ b/apps/agent-forge/server/__tests__/observeProxy.test.ts @@ -4,7 +4,8 @@ * headers. A dead daemon answers 502 with the `lousho observe start` hint. * Writes go through only when OBSERVE_WRITE_ALLOWLIST names the method and * path (feedback scores, a prompt label move, a hub publish, dataset and - * example writes, annotation queue writes); every other non-GET stays 405. + * example writes, annotation queue writes, evaluator rule writes); every + * other non-GET stays 405. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as http from 'node:http'; @@ -45,6 +46,12 @@ describe('observe proxy', () => { let body = ''; req.on('data', (chunk: Buffer) => (body += chunk.toString())); req.on('end', () => { + // A code rule, so judge.ts's test route (registered before the proxy) forwards its /test here. + if (req.method === 'GET' && req.url === '/api/v1/evaluators/r1') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ evaluator: { rule_id: 'r1', kind: 'code' } })); + return; + } if (req.method === 'DELETE') { lastDelete = { url: req.url, contentType: req.headers['content-type'] ?? null, body }; res.writeHead(204); @@ -191,6 +198,46 @@ describe('observe proxy', () => { expect(res.status).toBe(405); }); + it('forwards POST /api/v1/evaluators with its JSON body', async () => { + const body = { name: 'answered', kind: 'code', config: { source: '(trace) => 1' } }; + const res = await request(app).post('/observe/api/v1/evaluators').send(body); + expect(res.status).toBe(201); + expect(res.body).toEqual({ echoed: '/api/v1/evaluators', method: 'POST', contentType: 'application/json', body }); + }); + + it('forwards PUT /api/v1/evaluators/:id with its partial JSON body', async () => { + const res = await request(app).put('/observe/api/v1/evaluators/r1').send({ enabled: false }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ echoed: '/api/v1/evaluators/r1', method: 'PUT', contentType: 'application/json', body: { enabled: false } }); + }); + + it('forwards DELETE /api/v1/evaluators/:id with no body and passes the 204 back', async () => { + lastDelete = undefined; + const res = await request(app).delete('/observe/api/v1/evaluators/r1'); + expect(res.status).toBe(204); + expect(lastDelete).toEqual({ url: '/api/v1/evaluators/r1', contentType: null, body: '' }); + }); + + it('leaves POST /api/v1/evaluators/:id/test to the Studio route, which forwards a code rule to the daemon', async () => { + const res = await request(app).post('/observe/api/v1/evaluators/r1/test').send({ trace_id: 't1' }); + expect(res.status).toBe(201); + expect(res.body).toMatchObject({ echoed: '/api/v1/evaluators/r1/test', method: 'POST', body: { trace_id: 't1' } }); + }); + + it.each([ + ['delete', '/observe/api/v1/evaluators'], + ['put', '/observe/api/v1/evaluators'], + ['post', '/observe/api/v1/evaluators/r1'], + ['put', '/observe/api/v1/evaluators/r1/test'], + ['post', '/observe/api/v1/evaluators/judge-jobs/claim'], + ['post', '/observe/api/v1/evaluators/judge-jobs/j1/result'], + ['put', '/observe/api/v1/evaluators/%2e%2e'], + ['delete', '/observe/api/v1/evaluators/a%2Fb'], + ] as const)('keeps the evaluator-adjacent write %s %s at 405', async (method, url) => { + const res = await request(app)[method](url).send({}); + expect(res.status).toBe(405); + }); + it.each([ ['post', '/observe/api/v1/prompts'], ['put', '/observe/api/v1/prompts/mira'], @@ -224,8 +271,10 @@ describe('observe proxy', () => { expect(res.status).toBe(413); }); - it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes, dataset and queue writes', () => { - expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE', 'POST', 'POST', 'POST']); + it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes, dataset, queue and evaluator writes', () => { + expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual([ + 'POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE', 'POST', 'POST', 'POST', 'POST', 'PUT', 'DELETE', + ]); expect(OBSERVE_WRITE_ALLOWLIST.every((entry) => entry.path instanceof RegExp)).toBe(true); }); }); diff --git a/apps/agent-forge/server/buildAgent.ts b/apps/agent-forge/server/buildAgent.ts index ecac4d6f..020fd70f 100644 --- a/apps/agent-forge/server/buildAgent.ts +++ b/apps/agent-forge/server/buildAgent.ts @@ -31,7 +31,7 @@ import { } from '@lousho/build-ai-agent'; import type { AgentFlow } from '@lousho/build-ai-agent/flows'; import { compileHooksFromSpecPolicy } from './compileHooks'; -import { isSecretProvider, type SecretsStore } from './secretsStore'; +import { PROVIDER_KEY_ENV, isSecretProvider, type SecretsStore } from './secretsStore'; /** * Local, server-only tools available to a spec's `tools` list in ADDITION @@ -84,11 +84,6 @@ export interface BuiltAgent { sandbox: SandboxAdapter; } -const REAL_KEYED_PROVIDER_ENV: Record = { - openai: 'OPENAI_API_KEY', - anthropic: 'ANTHROPIC_API_KEY', -}; - /** Real provider types this app doesn't manage a stored key for (no plain-API-key concept, or not yet wired into secretsStore) - env-var-only. */ const REAL_ENV_ONLY_PROVIDERS = new Set(['ollama', 'openrouter']); @@ -121,8 +116,8 @@ function resolveProviderForSpec(type: string, model: string, secretsStore?: Secr if (isSecretProvider(lower)) { const storedKey = secretsStore?.getKey(lower); - const key = storedKey || process.env[REAL_KEYED_PROVIDER_ENV[lower]]; - if (!key) throw missingProviderKey(lower, REAL_KEYED_PROVIDER_ENV[lower]); + const key = storedKey || process.env[PROVIDER_KEY_ENV[lower]]; + if (!key) throw missingProviderKey(lower, PROVIDER_KEY_ENV[lower]); return LLMProviderRegistry.create(lower, { defaultModel: model, apiKey: key }); } diff --git a/apps/agent-forge/server/observeProxy.ts b/apps/agent-forge/server/observeProxy.ts index 6f7efaa7..4d0f2e47 100644 --- a/apps/agent-forge/server/observeProxy.ts +++ b/apps/agent-forge/server/observeProxy.ts @@ -12,10 +12,11 @@ * {@link OBSERVE_WRITE_ALLOWLIST} names the method and path - a short list of * Studio actions (human feedback scores, moving a prompt label, editing a * Context Hub fleet entry, dataset and example writes, annotation queue - * writes) - with the JSON body + * writes, online evaluator rule create/update/delete) - with the JSON body * app.ts's `express.json` already parsed, re-serialised and capped at 1 MiB * (a DELETE forwards no body, and its 204 passes back empty). Every other non-GET - the ingest endpoint above all - answers 405 - * and stays reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it + * and stays reporter-only. (`POST /observe/api/v1/evaluators/:id/test` never + * reaches this proxy: app.ts registers judge.ts's route for it first.) `/observe` sits in app.ts's API_PREFIXES, so it * carries the same per-launch token requirement as the rest of the studio API. */ import type { Express, Request } from 'express'; @@ -42,6 +43,9 @@ export const OBSERVE_WRITE_ALLOWLIST: readonly ObserveWrite[] = [ { method: 'POST', path: /^\/api\/v1\/queues$/ }, { method: 'POST', path: /^\/api\/v1\/queues\/[^/]+\/items$/ }, { method: 'POST', path: /^\/api\/v1\/queues\/[^/]+\/items\/[^/]+\/complete$/ }, + { method: 'POST', path: /^\/api\/v1\/evaluators$/ }, + { method: 'PUT', path: /^\/api\/v1\/evaluators\/[^/]+$/ }, + { method: 'DELETE', path: /^\/api\/v1\/evaluators\/[^/]+$/ }, ]; /** diff --git a/apps/agent-forge/server/secretsStore.test.ts b/apps/agent-forge/server/secretsStore.test.ts index 47aea624..7c3b2b94 100644 --- a/apps/agent-forge/server/secretsStore.test.ts +++ b/apps/agent-forge/server/secretsStore.test.ts @@ -8,20 +8,36 @@ describe('SecretsStore', () => { let baseDir: string; let store: SecretsStore; + const savedEnv = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY }; + beforeEach(() => { + delete process.env.OPENAI_API_KEY; + delete process.env.ANTHROPIC_API_KEY; baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lou-r-secrets-')); store = new SecretsStore(baseDir); }); afterEach(() => { + for (const [name, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } fs.rmSync(baseDir, { recursive: true, force: true }); }); it('reports no key stored for a fresh provider', () => { const status = store.list(); expect(status).toEqual([ - { provider: 'openai', hasKey: false, masked: null }, - { provider: 'anthropic', hasKey: false, masked: null }, + { provider: 'openai', hasKey: false, masked: null, envKey: false }, + { provider: 'anthropic', hasKey: false, masked: null, envKey: false }, + ]); + }); + + it('reports envKey from the provider env var, independent of a stored key', () => { + process.env.ANTHROPIC_API_KEY = 'env-key-placeholder'; + expect(store.list().map((s) => [s.provider, s.hasKey, s.envKey])).toEqual([ + ['openai', false, false], + ['anthropic', false, true], ]); }); diff --git a/apps/agent-forge/server/secretsStore.ts b/apps/agent-forge/server/secretsStore.ts index 791532fd..abcb2cfd 100644 --- a/apps/agent-forge/server/secretsStore.ts +++ b/apps/agent-forge/server/secretsStore.ts @@ -50,6 +50,12 @@ import type { ProviderKeyStatus } from '../shared/wireTypes'; const SECRET_PROVIDERS = ['openai', 'anthropic'] as const; export type SecretProvider = (typeof SECRET_PROVIDERS)[number]; +/** The env var each managed provider falls back to when no key is stored (buildAgent.ts resolves runs with it; `list()` reports whether it is set). */ +export const PROVIDER_KEY_ENV: Record = { + openai: 'OPENAI_API_KEY', + anthropic: 'ANTHROPIC_API_KEY', +}; + export function isSecretProvider(value: string): value is SecretProvider { return (SECRET_PROVIDERS as readonly string[]).includes(value); } @@ -169,13 +175,13 @@ export class SecretsStore { } } - /** Masked status for every managed provider - safe to return from a GET route. */ + /** Masked status for every managed provider, plus whether its env var is set (never its value) - safe to return from a GET route. */ list(): ProviderKeyStatus[] { const records = this.loadRecords(); return SECRET_PROVIDERS.map((provider) => { const hasKey = provider in records; const key = hasKey ? this.getKey(provider) : undefined; - return { provider, hasKey: hasKey && !!key, masked: key ? maskKey(key) : null }; + return { provider, hasKey: hasKey && !!key, masked: key ? maskKey(key) : null, envKey: !!process.env[PROVIDER_KEY_ENV[provider]] }; }); } } diff --git a/apps/agent-forge/shared/wireTypes.ts b/apps/agent-forge/shared/wireTypes.ts index ef5dd71b..5b94bbc2 100644 --- a/apps/agent-forge/shared/wireTypes.ts +++ b/apps/agent-forge/shared/wireTypes.ts @@ -216,6 +216,11 @@ export interface ProviderKeyStatus { hasKey: boolean; /** Last-4-visible masked form for display, e.g. '••••••••3f2a'. `null` when no key is stored. Never the real key. */ masked: string | null; + /** + * The provider's env var (`OPENAI_API_KEY` / `ANTHROPIC_API_KEY`) is set in the server's environment. Runs fall back + * to it when no key is stored, so a provider with only `envKey` can still run. Never the value. + */ + envKey: boolean; } /** diff --git a/apps/agent-forge/src/components/layout.css b/apps/agent-forge/src/components/layout.css index 61fca640..b6fd0f65 100644 --- a/apps/agent-forge/src/components/layout.css +++ b/apps/agent-forge/src/components/layout.css @@ -26,23 +26,6 @@ flex: 1; min-height: 0; } -.screen-placeholder { - padding: 32px 24px; - max-width: 560px; - color: var(--text-muted); -} -.screen-placeholder h2 { - margin: 0 0 8px; - font-size: 18px; - color: var(--text); -} -.screen-placeholder p { - margin: 0 0 16px; -} -.screen-placeholder a { - color: var(--accent); -} - /* ---------- Top bar ---------- */ .topbar { diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts index bf02daaa..f9355ff4 100644 --- a/apps/agent-forge/src/observe/observeClient.ts +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -11,7 +11,12 @@ */ import { STUDIO_TOKEN_HEADER, loadStudioToken } from '../runtime/studioToken'; import type { + EvaluatorPatch, + NewEvaluator, ObserveAgentRow, + ObserveEvaluator, + ObserveEvaluatorList, + ObserveEvaluatorTestResult, NewHubEntry, NewQueueScore, ObserveDataset, @@ -121,7 +126,7 @@ export interface ScoreListFilter { /** Eval name: every score named `` or `/`. */ eval?: string; agent_id?: string; - /** `eval` | `judge` | `human`. */ + /** `eval` | `judge` | `human` | `online`. */ source?: string; /** `created_at` lower bound (epoch ms). */ since?: number; @@ -153,6 +158,12 @@ export interface PageQuery { offset?: number; } +/** `GET /api/v1/evaluators` params: a page, optionally only one `kind` or only enabled/disabled rules. */ +export interface EvaluatorListFilter extends PageQuery { + kind?: 'code' | 'judge'; + enabled?: boolean; +} + /** The body of {@link ObserveClient.addExample}: a literal example, or one copied from a trace's root input/output. */ export type NewExample = { input: unknown; expected?: unknown } | { from_trace_id: string }; @@ -420,6 +431,55 @@ export class ObserveClient { }); } + /** `GET /api/v1/evaluators` - a page of online evaluator rules, oldest first. */ + listEvaluators(filter: EvaluatorListFilter = {}): Promise { + const { enabled, ...rest } = filter; + return this.request('/api/v1/evaluators', { query: { ...rest, enabled: enabled === undefined ? undefined : String(enabled) } }); + } + + /** `GET /api/v1/evaluators/:id` - one rule; `undefined` on the daemon's 404. */ + async evaluator(ruleId: string): Promise { + try { + const body = await this.request<{ evaluator: ObserveEvaluator }>(`/api/v1/evaluators/${encodeURIComponent(ruleId)}`); + return body.evaluator; + } catch (error) { + if (error instanceof ObserveApiError && error.status === 404) return undefined; + throw error; + } + } + + /** `POST /api/v1/evaluators` - a new rule; a taken name is the daemon's 409 (`ObserveApiError`). */ + async createEvaluator(input: NewEvaluator): Promise { + const body = await this.request<{ evaluator: ObserveEvaluator }>('/api/v1/evaluators', { method: 'POST', body: input }); + return body.evaluator; + } + + /** `PUT /api/v1/evaluators/:id` - a partial update; renaming onto a taken name is a 409. */ + async updateEvaluator(ruleId: string, patch: EvaluatorPatch): Promise { + const body = await this.request<{ evaluator: ObserveEvaluator }>(`/api/v1/evaluators/${encodeURIComponent(ruleId)}`, { + method: 'PUT', + body: patch, + }); + return body.evaluator; + } + + /** `DELETE /api/v1/evaluators/:id` - removes the rule (204); its past scores stay. */ + async deleteEvaluator(ruleId: string): Promise { + await this.request(`/api/v1/evaluators/${encodeURIComponent(ruleId)}`, { method: 'DELETE' }); + } + + /** + * `POST /api/v1/evaluators/:id/test {trace_id}` - a dry run of the rule on one trace (no score written). + * Same-origin this is the Studio server's route: it runs a judge rule with Studio's provider keys and forwards a code rule to the daemon. + */ + async testEvaluator(ruleId: string, traceId: string): Promise { + const body = await this.request<{ result: ObserveEvaluatorTestResult }>(`/api/v1/evaluators/${encodeURIComponent(ruleId)}/test`, { + method: 'POST', + body: { trace_id: traceId }, + }); + return body.result; + } + /** `GET /api/v1/prompts/:name/:ref` (ref = `latest`, a version number or a label); `undefined` on the daemon's 404. */ async getPrompt(name: string, ref = 'latest'): Promise { try { diff --git a/apps/agent-forge/src/observe/types.ts b/apps/agent-forge/src/observe/types.ts index dd3e848c..7a6c9862 100644 --- a/apps/agent-forge/src/observe/types.ts +++ b/apps/agent-forge/src/observe/types.ts @@ -331,3 +331,56 @@ export interface NewQueueScore { value: number; comment?: string; } + +/** Which traces an online evaluator runs on (every key optional; `{}` matches every trace). */ +export interface ObserveEvaluatorFilter { + agent_id?: string; + status?: 'ok' | 'error'; + /** Substring of the trace root's name. */ + name_contains?: string; +} + +/** A `code` rule's config (`(trace) => number | { value, comment }` source) or a `judge` rule's (`provider/model` or `mock`, plus a rubric). */ +export type ObserveEvaluatorKindConfig = + | { kind: 'code'; config: { source: string } } + | { kind: 'judge'; config: { model: string; rubric: string } }; + +/** The fields of an online evaluator rule a client sets (the body of `POST /api/v1/evaluators`). */ +export type NewEvaluator = ObserveEvaluatorKindConfig & { + name: string; + enabled?: boolean; + filter?: ObserveEvaluatorFilter; + /** 0..1, the share of matching traces the rule runs on. */ + sample_rate?: number; +}; + +/** One online evaluator rule (`GET /api/v1/evaluators` row, and the `evaluator` of the detail/create/update answers). */ +export type ObserveEvaluator = ObserveEvaluatorKindConfig & { + rule_id: string; + name: string; + enabled: boolean; + filter: ObserveEvaluatorFilter; + sample_rate: number; + created_at: number; + updated_at: number; +}; + +/** `PUT /api/v1/evaluators/:id`: any subset of the create fields (`kind` needs a matching `config`). */ +export type EvaluatorPatch = Partial> & { + kind?: ObserveEvaluatorKindConfig['kind']; + config?: ObserveEvaluatorKindConfig['config']; +}; + +/** `GET /api/v1/evaluators`: a page of rules, oldest first. */ +export interface ObserveEvaluatorList { + evaluators: ObserveEvaluator[]; + total: number; + limit: number; + offset: number; +} + +/** A dry run of one rule on one trace (`POST /api/v1/evaluators/:id/test`): no score is written. */ +export interface ObserveEvaluatorTestResult { + value: number | null; + comment: string | null; +} diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts index ad5c3d6f..cdd2923c 100644 --- a/apps/agent-forge/src/observe/useObserve.ts +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -6,11 +6,13 @@ * duplicating effect logic. */ import { useCallback, useEffect, useRef, useState } from 'react'; -import { observeClient, ObserveDaemonDownError, type HubEntryFilter, type InsightsFilter, type PageQuery, type ScoreListFilter, type SessionListFilter, type TraceListFilter } from './observeClient'; +import { observeClient, ObserveDaemonDownError, type EvaluatorListFilter, type HubEntryFilter, type InsightsFilter, type PageQuery, type ScoreListFilter, type SessionListFilter, type TraceListFilter } from './observeClient'; import type { ObserveAgentRow, ObserveDatasetDetail, ObserveDatasetList, + ObserveEvaluator, + ObserveEvaluatorList, ObserveQueue, ObserveQueueItem, ObserveQueueList, @@ -211,3 +213,18 @@ export function useQueue(queueId: string, refreshKey?: unknown): ObserveQuery { return useObserveQuery(() => observeClient.nextQueueItem(queueId), [queueId, refreshKey]); } + +/** `GET /api/v1/evaluators` - a page of online evaluator rules. */ +export function useEvaluators(filter: EvaluatorListFilter, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listEvaluators(filter), [JSON.stringify(filter), refreshKey]); +} + +/** `GET /api/v1/evaluators/:id` - one rule; `data` is `undefined` for an unknown id. */ +export function useEvaluator(ruleId: string, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.evaluator(ruleId), [ruleId, refreshKey]); +} + +/** The newest `limit` online-evaluator scores (`GET /api/v1/scores?source=online`), newest first. */ +export function useOnlineScores(limit: number, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listScores({ source: 'online', limit }), [limit, refreshKey]); +} diff --git a/apps/agent-forge/src/screens/Placeholder.tsx b/apps/agent-forge/src/screens/Placeholder.tsx deleted file mode 100644 index abed0cd1..00000000 --- a/apps/agent-forge/src/screens/Placeholder.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import type { ReactNode } from 'react'; - -/** Stand-in for a screen that has a nav item but no content yet: what it is, what's coming, and a way back; `nav` is an optional sub-navigation above it. */ -export function Placeholder({ title, blurb, nav }: { title: string; blurb: string; nav?: ReactNode }) { - return ( -
- {nav} -

{title}

-

{blurb}

- Back to Agents -
- ); -} diff --git a/apps/agent-forge/src/screens/__tests__/routes.test.tsx b/apps/agent-forge/src/screens/__tests__/routes.test.tsx index c416542f..3124dc6f 100644 --- a/apps/agent-forge/src/screens/__tests__/routes.test.tsx +++ b/apps/agent-forge/src/screens/__tests__/routes.test.tsx @@ -21,7 +21,6 @@ vi.mock('../../state/AppState', () => ({ useAppState: () => appState })); import { screenFor, useScreen } from '../routes'; import { ROUTES } from '../routeTable'; -import { Placeholder } from '../Placeholder'; import { EvalsTabs } from '../evals/EvalsTabs'; import { Topbar } from '../../components/Topbar'; @@ -66,7 +65,8 @@ describe('screenFor', () => { expect(screenFor('#/evals/scores')).toBe('scores'); expect(screenFor('#/evals/datasets')).toBe('datasets'); expect(screenFor('#/evals/datasets/golden.v1?offset=50')).toBe('datasets'); - expect(screenFor('#/evals/evaluators')).toBe('evals'); + expect(screenFor('#/evals/evaluators')).toBe('evaluators'); + expect(screenFor('#/evals/evaluators/r1?trace=t1')).toBe('evaluators'); expect(screenFor('#/evals/queues')).toBe('queues'); expect(screenFor('#/evals/queues/q1')).toBe('queues'); expect(screenFor('#/playground')).toBe('playground'); @@ -79,6 +79,9 @@ describe('screenFor', () => { expect(screenFor('')).toBeNull(); expect(screenFor('#/')).toBeNull(); expect(screenFor('#/nope')).toBeNull(); + // Every Evaluations section has its own screen now; an unknown one is an unknown route. + expect(screenFor('#/evals/nope')).toBeNull(); + expect(screenFor('#/evals/nope/1')).toBeNull(); }); it("gives every route an example path that lands on that route's screen", () => { for (const { pattern, screen, example } of ROUTES) { @@ -171,15 +174,6 @@ describe('Topbar', () => { }); }); -describe('Placeholder', () => { - it('shows a title, what is coming, and a link back to Agents', () => { - render(); - expect(container.querySelector('h2')?.textContent).toBe('Playground'); - expect(container.textContent).toContain('Coming soon.'); - expect(container.querySelector('a[href="#/agents"]')).not.toBeNull(); - }); -}); - describe('EvalsTabs', () => { it('links every Evaluations screen and marks the current one', () => { window.history.replaceState(null, '', '/?token=secret#/evals/datasets/golden'); diff --git a/apps/agent-forge/src/screens/evals/evals.css b/apps/agent-forge/src/screens/evals/evals.css index 8b5fc78e..4acb5542 100644 --- a/apps/agent-forge/src/screens/evals/evals.css +++ b/apps/agent-forge/src/screens/evals/evals.css @@ -5,10 +5,7 @@ gap: 4px; border-bottom: 1px solid var(--border); } -.screen-placeholder .evals-tabs { - margin-bottom: 16px; -} -/* Two classes: outranks a screen's own `a` colour (the placeholder's accent links). */ +/* Two classes: outranks a screen's own `a` colour. */ .evals-tabs .evals-tab { padding: 6px 10px; border-bottom: 2px solid transparent; diff --git a/apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx b/apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx new file mode 100644 index 00000000..396836a0 --- /dev/null +++ b/apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx @@ -0,0 +1,152 @@ +/** + * `#/evals/evaluators/:id` - one online evaluator rule: the shared form (a + * save sends only the changed fields, `PUT /api/v1/evaluators/:id`), Delete + * (after a confirm), and "Test on trace" - a dry run of the rule on one trace + * through the Studio server (`POST /observe/api/v1/evaluators/:id/test`, + * which runs a judge with Studio's provider keys and forwards a code rule to + * the daemon), showing the value and comment; no score is written. + * `?trace=` prefills the trace id. + */ +import { useState, type FormEvent } from 'react'; +import { navigate } from '../../router/router'; +import { useRoute } from '../../router/useRoute'; +import { ObserveDaemonDown, ObserveError } from '../../observe/chrome'; +import { ObserveApiError, observeClient } from '../../observe/observeClient'; +import type { NewEvaluator, ObserveEvaluator, ObserveEvaluatorTestResult } from '../../observe/types'; +import { useAgents, useDaemonDown, useEvaluator } from '../../observe/useObserve'; +import { EmptyState } from '../../ui/EmptyState'; +import { useToast } from '../../ui/Toast'; +import { useProviderKeys } from '../playground/ModelPicker'; +import { draftFrom, evaluatorPatch } from './evaluatorDraft'; +import { EvaluatorForm } from './EvaluatorForm'; +import { evaluatorsPath, parseListState } from './evaluatorsUrl'; + +/** "Test on trace": a trace id, a Test button, and the dry run's value and comment. */ +function TestOnTrace({ rule, prefill }: { rule: ObserveEvaluator; prefill: string }) { + const toast = useToast(); + const [traceId, setTraceId] = useState(prefill); + const [result, setResult] = useState(); + const [notFound, setNotFound] = useState(false); + const [running, setRunning] = useState(false); + + const run = async (e: FormEvent) => { + e.preventDefault(); + const id = traceId.trim(); + if (!id) return; + setRunning(true); + setResult(undefined); + setNotFound(false); + try { + setResult(await observeClient.testEvaluator(rule.rule_id, id)); + } catch (failure) { + if (failure instanceof ObserveApiError && failure.status === 404) setNotFound(true); + else toast.show({ tone: 'error', message: `Test failed: ${(failure as Error).message}` }); + } finally { + setRunning(false); + } + }; + + return ( +
void run(e)}> +

Test on trace

+

Runs this rule once on one trace. No score is written.

+
+ + +
+ {notFound && ( +

+ Trace not found +

+ )} + {result && ( +
+
Value
+
{result.value === null ? 'null' : String(result.value)}
+
Comment
+
{result.comment ?? '-'}
+
+ )} +
+ ); +} + +/** The edit form, Delete and Test on trace for a loaded rule; a save or delete reports through a toast. */ +function EditRule({ rule, back, onSaved }: { rule: ObserveEvaluator; back: string; onSaved: () => void }) { + const toast = useToast(); + const keys = useProviderKeys(); + const agents = useAgents(); + const { query } = useRoute(); + + const save = async (body: NewEvaluator) => { + const patch = evaluatorPatch(rule, body); + if (Object.keys(patch).length === 0) { + toast.show({ message: 'No changes to save.' }); + return false; + } + await observeClient.updateEvaluator(rule.rule_id, patch); + toast.show({ tone: 'success', message: `Saved evaluator ${body.name}.` }); + onSaved(); + return false; + }; + const remove = async () => { + if (!window.confirm(`Delete evaluator ${rule.name}? Its past scores stay.`)) return; + try { + await observeClient.deleteEvaluator(rule.rule_id); + toast.show({ tone: 'success', message: `Deleted evaluator ${rule.name}.` }); + navigate(back); + } catch (failure) { + toast.show({ tone: 'error', message: `Could not delete ${rule.name}: ${(failure as Error).message}` }); + } + }; + + return ( + <> + void remove()}> + Delete + + } + /> + + + ); +} + +/** `#/evals/evaluators/:id`; an unknown rule shows an empty state with the way back. */ +export function EvaluatorDetail({ ruleId }: { ruleId: string }) { + const { query } = useRoute(); + const [refreshKey, setRefreshKey] = useState(0); + const rule = useEvaluator(ruleId, refreshKey); + const back = evaluatorsPath(parseListState(query)); + const backLink = All evaluators; + if (useDaemonDown(false, [rule])) return ; + let body; + if (rule.loading && !rule.data) body =

Loading evaluator...

; + else if (rule.error) body = ; + else if (!rule.data) body = ; + else body = setRefreshKey((k) => k + 1)} />; + return ( + <> + +

Evaluator {rule.data ? {rule.data.name} : null}

+ {body} + + ); +} diff --git a/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx new file mode 100644 index 00000000..4081377e --- /dev/null +++ b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx @@ -0,0 +1,240 @@ +/** + * The evaluator form the list's "New evaluator" and the edit view share: + * name, kind (`code` with a function textarea, or `judge` with a model and a + * rubric), the trace filter (agent, status, name contains), the sample rate + * and the enabled flag. A judge needs a provider key, stored in Studio's + * settings or set as the provider's env var where Studio runs (judges run + * with either): without one the judge option is disabled, with a hint that + * links Settings. + * A failed save (a taken name is the daemon's 409) shows inline. + */ +import { useState, type FormEvent, type ReactNode } from 'react'; +import type { ProviderKeyStatus } from '../../../shared/wireTypes'; +import { ObserveApiError } from '../../observe/observeClient'; +import type { NewEvaluator, ObserveAgentRow } from '../../observe/types'; +import { MOCK_PROVIDER, defaultProvider, providerOptions } from '../playground/request'; +import { draftProblem, toNewEvaluator, type EvaluatorDraft } from './evaluatorDraft'; + +type SetField = (field: K, value: EvaluatorDraft[K]) => void; + +/** The save failure shown inline: a 409 is a taken name, anything else its message. */ +const saveError = (error: unknown): string => + error instanceof ObserveApiError && error.status === 409 + ? 'Could not save: an evaluator with that name exists.' + : `Could not save the evaluator: ${(error as Error).message}`; + +/** The providers a judge can run on: a key stored in Settings or the provider's env var (`envKey`) both count. */ +const judgeKeys = (keys: readonly ProviderKeyStatus[]): ProviderKeyStatus[] => keys.map((k) => ({ ...k, hasKey: k.hasKey || k.envKey })); + +function KindField({ + draft, + choose, + judgeAllowed, +}: { + draft: EvaluatorDraft; + choose: (kind: EvaluatorDraft['kind']) => void; + judgeAllowed: boolean; +}) { + return ( +
+ Kind + + + {!judgeAllowed && ( +

+ Add a provider key in Settings to use judge evaluators. +

+ )} +
+ ); +} + +function FilterFields({ draft, set, agents }: { draft: EvaluatorDraft; set: SetField; agents: ObserveAgentRow[] }) { + const known = agents.some((a) => a.agent_id === draft.agentId); + return ( +
+ Runs on traces where + + + +
+ ); +} + +function JudgeFields({ draft, set, keys }: { draft: EvaluatorDraft; set: SetField; keys: readonly ProviderKeyStatus[] }) { + const options = providerOptions(keys); + if (draft.provider !== '' && !options.includes(draft.provider)) options.unshift(draft.provider); + const isMock = draft.provider === MOCK_PROVIDER; + return ( + <> +
+ + +
+