From 94b6e82ac092a12d15f1ee92c284dcf7c9858a2b Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 29 Oct 2026 02:51:05 +0300 Subject: [PATCH 1/2] feat(studio): online evaluators #/evals/evaluators lists the observe daemon's online evaluator rules (kind, filter summary, sample rate, an enabled switch that PUTs {enabled} optimistically and rolls back with an error toast, last run and runs in the last 24h from one source=online score read grouped by rule name). "New evaluator" opens a form for code rules (a monospace textarea prefilled with a sample valid against the documented trace object) and judge rules (provider/model + rubric); without a Studio provider key the judge option is disabled with a hint linking Settings. #/evals/evaluators/:id edits one rule with the same form (a PUT holding only the changed fields; a taken name shows inline), deletes it after a confirm, and tests it on a trace through the Studio route POST /observe/api/v1/evaluators/:id/test (value + comment; 404 reads "Trace not found"; ?trace= prefills). List filters ?kind=&enabled= and ?offset= live in the URL and ride along to the edit view. The proxy allowlist gains POST /api/v1/evaluators and PUT/DELETE /api/v1/evaluators/:id. Every Evaluations section now has a screen, so the generic /evals/:section routes and the Placeholder are gone (an unknown section is an unknown route). e2e: evaluator fixtures in observeStub; revamp.spec checks the evaluator list and edit controls. Co-Authored-By: Claude Opus 5.5 --- apps/agent-forge/e2e/a11y.spec.ts | 4 +- apps/agent-forge/e2e/observeStub.ts | 40 +- apps/agent-forge/e2e/revamp.spec.ts | 14 +- .../server/__tests__/observeProxy.test.ts | 55 ++- apps/agent-forge/server/observeProxy.ts | 8 +- apps/agent-forge/src/components/layout.css | 17 - apps/agent-forge/src/observe/observeClient.ts | 62 ++- apps/agent-forge/src/observe/types.ts | 53 +++ apps/agent-forge/src/observe/useObserve.ts | 19 +- apps/agent-forge/src/screens/Placeholder.tsx | 13 - .../src/screens/__tests__/routes.test.tsx | 16 +- apps/agent-forge/src/screens/evals/evals.css | 5 +- .../screens/evaluators/EvaluatorDetail.tsx | 152 +++++++ .../src/screens/evaluators/EvaluatorForm.tsx | 234 +++++++++++ .../screens/evaluators/EvaluatorsScreen.tsx | 226 ++++++++++ .../__tests__/EvaluatorsScreen.test.tsx | 395 ++++++++++++++++++ .../__tests__/evaluatorsUrl.test.ts | 81 ++++ .../src/screens/evaluators/evaluatorDraft.ts | 119 ++++++ .../src/screens/evaluators/evaluators.css | 75 ++++ .../src/screens/evaluators/evaluatorsUrl.ts | 106 +++++ apps/agent-forge/src/screens/routeTable.ts | 6 +- apps/agent-forge/src/screens/routes.tsx | 12 +- 22 files changed, 1642 insertions(+), 70 deletions(-) delete mode 100644 apps/agent-forge/src/screens/Placeholder.tsx create mode 100644 apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx create mode 100644 apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx create mode 100644 apps/agent-forge/src/screens/evaluators/EvaluatorsScreen.tsx create mode 100644 apps/agent-forge/src/screens/evaluators/__tests__/EvaluatorsScreen.test.tsx create mode 100644 apps/agent-forge/src/screens/evaluators/__tests__/evaluatorsUrl.test.ts create mode 100644 apps/agent-forge/src/screens/evaluators/evaluatorDraft.ts create mode 100644 apps/agent-forge/src/screens/evaluators/evaluators.css create mode 100644 apps/agent-forge/src/screens/evaluators/evaluatorsUrl.ts diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index b04f24985..668eb7a77 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -128,8 +128,8 @@ async function expectNoSeriousAxeViolations(page: Page, label: string) { // Every route in the table (new screens are covered automatically): one
, one h1, // the banner and the primary nav, and an axe scan of the populated (stubbed daemon) state. // Plus the deep-link states that render a different view on the same route. -// The datasets and queues routes scan the stub's dataset and annotation queue fixtures. Placeholder routes (/evals/:section, /evals/:section/:id) scan the Placeholder today; when their real -// screens land, add their endpoints to e2e/observeStub.ts or they get scanned in a 404/empty state. +// The datasets, queues and evaluators routes scan the stub's dataset, annotation queue and evaluator fixtures; a new +// screen's endpoints go in e2e/observeStub.ts too, or it gets scanned in a 404/empty state. const SCANS = [ ...ROUTES.map(({ pattern, example }) => ({ label: pattern, path: example })), ...['/traces?status=error', '/traces/e2e-trace?obs=e2e-obs', '/threads?status=active', '/prompts/e2e-prompt?diff=1,2'].map((path) => ({ label: path, path })), diff --git a/apps/agent-forge/e2e/observeStub.ts b/apps/agent-forge/e2e/observeStub.ts index 8c6f8b7d2..67bd0ed6d 100644 --- a/apps/agent-forge/e2e/observeStub.ts +++ b/apps/agent-forge/e2e/observeStub.ts @@ -2,6 +2,7 @@ import type { Page, Route } from '@playwright/test'; import type { ObserveAgentRow, ObserveDataset, + ObserveEvaluator, ObserveExample, ObserveHubEntry, ObserveInsights, @@ -20,7 +21,8 @@ import type { * thread with one trace (an LLM call and a tool call), one prompt with two * versions, an eval's scores, a Context Hub namespace with a fleet and a * private entry, a dataset with two examples (one copied from the trace), and - * an annotation queue whose next pending item is the trace. + * an annotation queue whose next pending item is the trace, and two online + * evaluator rules (a code rule with online scores, and a judge rule). */ const NOW = Date.now(); const T0 = NOW - 60_000; @@ -225,6 +227,35 @@ const queueItem: ObserveQueueItem = { done_at: null, }; +const evaluators: ObserveEvaluator[] = [ + { + rule_id: 'e2e-evaluator', + name: 'answered', + enabled: true, + filter: { agent_id: 'e2e-agent', status: 'ok' }, + sample_rate: 1, + kind: 'code', + config: { source: "(trace) => (trace.output ? 1 : 0)" }, + created_at: T0 - HOUR, + updated_at: T0 - HOUR, + }, + { + rule_id: 'e2e-judge', + name: 'polite', + enabled: false, + filter: {}, + sample_rate: 0.5, + kind: 'judge', + config: { model: 'openai/gpt-4o-mini', rubric: 'Is the reply polite?' }, + created_at: T0 - HOUR, + updated_at: T0 - HOUR, + }, +]; +const onlineScores: ObserveScoreRow[] = [ + { ...score, score_id: 'e2e-online-1', name: 'answered', source: 'online', value: 1, comment: null, obs_id: null }, + { ...score, score_id: 'e2e-online-2', name: 'answered', source: 'online', value: 0, comment: null, obs_id: null, created_at: T0 - HOUR }, +]; + const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); @@ -260,9 +291,16 @@ export async function stubObserve(page: Page): Promise { return json(route, { queues: [queue], total: 1, limit: Number(q.get('limit') ?? 100), offset: 0 }); if (path === '/api/v1/queues/e2e-queue') return json(route, { queue }); if (path === '/api/v1/queues/e2e-queue/next') return json(route, { item: queueItem }); + if (path === '/api/v1/evaluators' && route.request().method() === 'GET') { + const rows = evaluators.filter((e) => (!q.get('kind') || e.kind === q.get('kind')) && (!q.get('enabled') || String(e.enabled) === q.get('enabled'))); + return json(route, { evaluators: rows, total: rows.length, limit: Number(q.get('limit') ?? 100), offset: 0 }); + } + if (path === '/api/v1/evaluators/e2e-evaluator' && route.request().method() === 'GET') return json(route, { evaluator: evaluators[0] }); + if (path === '/api/v1/evaluators/e2e-evaluator/test') return json(route, { result: { value: 1, comment: '1 tool call(s)' } }); if (path === '/api/v1/scores') { if (q.get('aggregate') === 'eval') return json(route, { evals: [{ name: 'e2e-eval', count: 4, passed: 3, pass_rate: 0.75, avg_value: 0.75, last_ts: T0 + 2_000 }] }); + if (q.get('source') === 'online') return json(route, { scores: onlineScores }); return json(route, { scores: [score, { ...score, score_id: 'e2e-score-2', value: 0, comment: 'wrong order id' }] }); } return json(route, { error: 'not found' }, 404); diff --git a/apps/agent-forge/e2e/revamp.spec.ts b/apps/agent-forge/e2e/revamp.spec.ts index c8a6da1b8..19a0cbe34 100644 --- a/apps/agent-forge/e2e/revamp.spec.ts +++ b/apps/agent-forge/e2e/revamp.spec.ts @@ -127,7 +127,8 @@ const FILTERS: Record = { datasets: '?offset=50', // Queues pages server-side the same way (`?offset=`). queues: '?offset=50', - evals: '', + // The list's kind/state filters (ridden along on the edit view for its way back) and the edit view's test-on-trace prefill. + evaluators: '?kind=code&enabled=true&trace=e2e-trace', prompts: '?diff=1,2', hub: '?agent=triage-bot&scope=fleet', settings: '', @@ -200,6 +201,15 @@ const CONTROLS: Record Promise> = { '/evals/queues': async (main) => { await expectPreviousPage(main); }, + '/evals/evaluators': async (main) => { + await expect(main.getByRole('combobox', { name: 'Kind' })).toHaveValue('code'); + await expect(main.getByRole('combobox', { name: 'State' })).toHaveValue('true'); + await expect(main.getByRole('switch', { name: 'answered enabled' })).toBeChecked(); + }, + '/evals/evaluators/:id': async (main) => { + await expect(main.getByRole('textbox', { name: 'Trace id' })).toHaveValue('e2e-trace'); + await expect(main.getByRole('link', { name: 'All evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true'); + }, '/prompts/:name': async (main) => { await expect(main.getByRole('combobox', { name: 'Compare version A' })).toHaveValue('1'); await expect(main.getByRole('combobox', { name: 'Compare version B' })).toHaveValue('2'); @@ -221,8 +231,6 @@ CONTROLS['/hub/:namespace'] = CONTROLS['/hub']; const NO_CONTROLS = new Set([ '/agents', // canonicalises to the open agent; nothing to filter '/threads/:sessionId', // a thread's turns; the list's filters only ride along in the URL - '/evals/:section', // evaluators / annotation placeholder screen - '/evals/:section/:id', // evaluators / annotation placeholder screen '/evals/queues/:id', // one queue's review pane; no URL-backed controls beyond the id '/prompts', // the registry list has no filters '/settings', // no URL state diff --git a/apps/agent-forge/server/__tests__/observeProxy.test.ts b/apps/agent-forge/server/__tests__/observeProxy.test.ts index 95713b13c..ed400077e 100644 --- a/apps/agent-forge/server/__tests__/observeProxy.test.ts +++ b/apps/agent-forge/server/__tests__/observeProxy.test.ts @@ -4,7 +4,8 @@ * headers. A dead daemon answers 502 with the `lousho observe start` hint. * Writes go through only when OBSERVE_WRITE_ALLOWLIST names the method and * path (feedback scores, a prompt label move, a hub publish, dataset and - * example writes, annotation queue writes); every other non-GET stays 405. + * example writes, annotation queue writes, evaluator rule writes); every + * other non-GET stays 405. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as http from 'node:http'; @@ -45,6 +46,12 @@ describe('observe proxy', () => { let body = ''; req.on('data', (chunk: Buffer) => (body += chunk.toString())); req.on('end', () => { + // A code rule, so judge.ts's test route (registered before the proxy) forwards its /test here. + if (req.method === 'GET' && req.url === '/api/v1/evaluators/r1') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ evaluator: { rule_id: 'r1', kind: 'code' } })); + return; + } if (req.method === 'DELETE') { lastDelete = { url: req.url, contentType: req.headers['content-type'] ?? null, body }; res.writeHead(204); @@ -191,6 +198,46 @@ describe('observe proxy', () => { expect(res.status).toBe(405); }); + it('forwards POST /api/v1/evaluators with its JSON body', async () => { + const body = { name: 'answered', kind: 'code', config: { source: '(trace) => 1' } }; + const res = await request(app).post('/observe/api/v1/evaluators').send(body); + expect(res.status).toBe(201); + expect(res.body).toEqual({ echoed: '/api/v1/evaluators', method: 'POST', contentType: 'application/json', body }); + }); + + it('forwards PUT /api/v1/evaluators/:id with its partial JSON body', async () => { + const res = await request(app).put('/observe/api/v1/evaluators/r1').send({ enabled: false }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ echoed: '/api/v1/evaluators/r1', method: 'PUT', contentType: 'application/json', body: { enabled: false } }); + }); + + it('forwards DELETE /api/v1/evaluators/:id with no body and passes the 204 back', async () => { + lastDelete = undefined; + const res = await request(app).delete('/observe/api/v1/evaluators/r1'); + expect(res.status).toBe(204); + expect(lastDelete).toEqual({ url: '/api/v1/evaluators/r1', contentType: null, body: '' }); + }); + + it('leaves POST /api/v1/evaluators/:id/test to the Studio route, which forwards a code rule to the daemon', async () => { + const res = await request(app).post('/observe/api/v1/evaluators/r1/test').send({ trace_id: 't1' }); + expect(res.status).toBe(201); + expect(res.body).toMatchObject({ echoed: '/api/v1/evaluators/r1/test', method: 'POST', body: { trace_id: 't1' } }); + }); + + it.each([ + ['delete', '/observe/api/v1/evaluators'], + ['put', '/observe/api/v1/evaluators'], + ['post', '/observe/api/v1/evaluators/r1'], + ['put', '/observe/api/v1/evaluators/r1/test'], + ['post', '/observe/api/v1/evaluators/judge-jobs/claim'], + ['post', '/observe/api/v1/evaluators/judge-jobs/j1/result'], + ['put', '/observe/api/v1/evaluators/%2e%2e'], + ['delete', '/observe/api/v1/evaluators/a%2Fb'], + ] as const)('keeps the evaluator-adjacent write %s %s at 405', async (method, url) => { + const res = await request(app)[method](url).send({}); + expect(res.status).toBe(405); + }); + it.each([ ['post', '/observe/api/v1/prompts'], ['put', '/observe/api/v1/prompts/mira'], @@ -224,8 +271,10 @@ describe('observe proxy', () => { expect(res.status).toBe(413); }); - it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes, dataset and queue writes', () => { - expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE', 'POST', 'POST', 'POST']); + it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes, dataset, queue and evaluator writes', () => { + expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual([ + 'POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE', 'POST', 'POST', 'POST', 'POST', 'PUT', 'DELETE', + ]); expect(OBSERVE_WRITE_ALLOWLIST.every((entry) => entry.path instanceof RegExp)).toBe(true); }); }); diff --git a/apps/agent-forge/server/observeProxy.ts b/apps/agent-forge/server/observeProxy.ts index 6f7efaa7c..4d0f2e470 100644 --- a/apps/agent-forge/server/observeProxy.ts +++ b/apps/agent-forge/server/observeProxy.ts @@ -12,10 +12,11 @@ * {@link OBSERVE_WRITE_ALLOWLIST} names the method and path - a short list of * Studio actions (human feedback scores, moving a prompt label, editing a * Context Hub fleet entry, dataset and example writes, annotation queue - * writes) - with the JSON body + * writes, online evaluator rule create/update/delete) - with the JSON body * app.ts's `express.json` already parsed, re-serialised and capped at 1 MiB * (a DELETE forwards no body, and its 204 passes back empty). Every other non-GET - the ingest endpoint above all - answers 405 - * and stays reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it + * and stays reporter-only. (`POST /observe/api/v1/evaluators/:id/test` never + * reaches this proxy: app.ts registers judge.ts's route for it first.) `/observe` sits in app.ts's API_PREFIXES, so it * carries the same per-launch token requirement as the rest of the studio API. */ import type { Express, Request } from 'express'; @@ -42,6 +43,9 @@ export const OBSERVE_WRITE_ALLOWLIST: readonly ObserveWrite[] = [ { method: 'POST', path: /^\/api\/v1\/queues$/ }, { method: 'POST', path: /^\/api\/v1\/queues\/[^/]+\/items$/ }, { method: 'POST', path: /^\/api\/v1\/queues\/[^/]+\/items\/[^/]+\/complete$/ }, + { method: 'POST', path: /^\/api\/v1\/evaluators$/ }, + { method: 'PUT', path: /^\/api\/v1\/evaluators\/[^/]+$/ }, + { method: 'DELETE', path: /^\/api\/v1\/evaluators\/[^/]+$/ }, ]; /** diff --git a/apps/agent-forge/src/components/layout.css b/apps/agent-forge/src/components/layout.css index 61fca6401..b6fd0f659 100644 --- a/apps/agent-forge/src/components/layout.css +++ b/apps/agent-forge/src/components/layout.css @@ -26,23 +26,6 @@ flex: 1; min-height: 0; } -.screen-placeholder { - padding: 32px 24px; - max-width: 560px; - color: var(--text-muted); -} -.screen-placeholder h2 { - margin: 0 0 8px; - font-size: 18px; - color: var(--text); -} -.screen-placeholder p { - margin: 0 0 16px; -} -.screen-placeholder a { - color: var(--accent); -} - /* ---------- Top bar ---------- */ .topbar { diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts index bf02daaaa..f9355ff45 100644 --- a/apps/agent-forge/src/observe/observeClient.ts +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -11,7 +11,12 @@ */ import { STUDIO_TOKEN_HEADER, loadStudioToken } from '../runtime/studioToken'; import type { + EvaluatorPatch, + NewEvaluator, ObserveAgentRow, + ObserveEvaluator, + ObserveEvaluatorList, + ObserveEvaluatorTestResult, NewHubEntry, NewQueueScore, ObserveDataset, @@ -121,7 +126,7 @@ export interface ScoreListFilter { /** Eval name: every score named `` or `/`. */ eval?: string; agent_id?: string; - /** `eval` | `judge` | `human`. */ + /** `eval` | `judge` | `human` | `online`. */ source?: string; /** `created_at` lower bound (epoch ms). */ since?: number; @@ -153,6 +158,12 @@ export interface PageQuery { offset?: number; } +/** `GET /api/v1/evaluators` params: a page, optionally only one `kind` or only enabled/disabled rules. */ +export interface EvaluatorListFilter extends PageQuery { + kind?: 'code' | 'judge'; + enabled?: boolean; +} + /** The body of {@link ObserveClient.addExample}: a literal example, or one copied from a trace's root input/output. */ export type NewExample = { input: unknown; expected?: unknown } | { from_trace_id: string }; @@ -420,6 +431,55 @@ export class ObserveClient { }); } + /** `GET /api/v1/evaluators` - a page of online evaluator rules, oldest first. */ + listEvaluators(filter: EvaluatorListFilter = {}): Promise { + const { enabled, ...rest } = filter; + return this.request('/api/v1/evaluators', { query: { ...rest, enabled: enabled === undefined ? undefined : String(enabled) } }); + } + + /** `GET /api/v1/evaluators/:id` - one rule; `undefined` on the daemon's 404. */ + async evaluator(ruleId: string): Promise { + try { + const body = await this.request<{ evaluator: ObserveEvaluator }>(`/api/v1/evaluators/${encodeURIComponent(ruleId)}`); + return body.evaluator; + } catch (error) { + if (error instanceof ObserveApiError && error.status === 404) return undefined; + throw error; + } + } + + /** `POST /api/v1/evaluators` - a new rule; a taken name is the daemon's 409 (`ObserveApiError`). */ + async createEvaluator(input: NewEvaluator): Promise { + const body = await this.request<{ evaluator: ObserveEvaluator }>('/api/v1/evaluators', { method: 'POST', body: input }); + return body.evaluator; + } + + /** `PUT /api/v1/evaluators/:id` - a partial update; renaming onto a taken name is a 409. */ + async updateEvaluator(ruleId: string, patch: EvaluatorPatch): Promise { + const body = await this.request<{ evaluator: ObserveEvaluator }>(`/api/v1/evaluators/${encodeURIComponent(ruleId)}`, { + method: 'PUT', + body: patch, + }); + return body.evaluator; + } + + /** `DELETE /api/v1/evaluators/:id` - removes the rule (204); its past scores stay. */ + async deleteEvaluator(ruleId: string): Promise { + await this.request(`/api/v1/evaluators/${encodeURIComponent(ruleId)}`, { method: 'DELETE' }); + } + + /** + * `POST /api/v1/evaluators/:id/test {trace_id}` - a dry run of the rule on one trace (no score written). + * Same-origin this is the Studio server's route: it runs a judge rule with Studio's provider keys and forwards a code rule to the daemon. + */ + async testEvaluator(ruleId: string, traceId: string): Promise { + const body = await this.request<{ result: ObserveEvaluatorTestResult }>(`/api/v1/evaluators/${encodeURIComponent(ruleId)}/test`, { + method: 'POST', + body: { trace_id: traceId }, + }); + return body.result; + } + /** `GET /api/v1/prompts/:name/:ref` (ref = `latest`, a version number or a label); `undefined` on the daemon's 404. */ async getPrompt(name: string, ref = 'latest'): Promise { try { diff --git a/apps/agent-forge/src/observe/types.ts b/apps/agent-forge/src/observe/types.ts index dd3e848cb..7a6c98629 100644 --- a/apps/agent-forge/src/observe/types.ts +++ b/apps/agent-forge/src/observe/types.ts @@ -331,3 +331,56 @@ export interface NewQueueScore { value: number; comment?: string; } + +/** Which traces an online evaluator runs on (every key optional; `{}` matches every trace). */ +export interface ObserveEvaluatorFilter { + agent_id?: string; + status?: 'ok' | 'error'; + /** Substring of the trace root's name. */ + name_contains?: string; +} + +/** A `code` rule's config (`(trace) => number | { value, comment }` source) or a `judge` rule's (`provider/model` or `mock`, plus a rubric). */ +export type ObserveEvaluatorKindConfig = + | { kind: 'code'; config: { source: string } } + | { kind: 'judge'; config: { model: string; rubric: string } }; + +/** The fields of an online evaluator rule a client sets (the body of `POST /api/v1/evaluators`). */ +export type NewEvaluator = ObserveEvaluatorKindConfig & { + name: string; + enabled?: boolean; + filter?: ObserveEvaluatorFilter; + /** 0..1, the share of matching traces the rule runs on. */ + sample_rate?: number; +}; + +/** One online evaluator rule (`GET /api/v1/evaluators` row, and the `evaluator` of the detail/create/update answers). */ +export type ObserveEvaluator = ObserveEvaluatorKindConfig & { + rule_id: string; + name: string; + enabled: boolean; + filter: ObserveEvaluatorFilter; + sample_rate: number; + created_at: number; + updated_at: number; +}; + +/** `PUT /api/v1/evaluators/:id`: any subset of the create fields (`kind` needs a matching `config`). */ +export type EvaluatorPatch = Partial> & { + kind?: ObserveEvaluatorKindConfig['kind']; + config?: ObserveEvaluatorKindConfig['config']; +}; + +/** `GET /api/v1/evaluators`: a page of rules, oldest first. */ +export interface ObserveEvaluatorList { + evaluators: ObserveEvaluator[]; + total: number; + limit: number; + offset: number; +} + +/** A dry run of one rule on one trace (`POST /api/v1/evaluators/:id/test`): no score is written. */ +export interface ObserveEvaluatorTestResult { + value: number | null; + comment: string | null; +} diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts index ad5c3d6f9..cdd2923cd 100644 --- a/apps/agent-forge/src/observe/useObserve.ts +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -6,11 +6,13 @@ * duplicating effect logic. */ import { useCallback, useEffect, useRef, useState } from 'react'; -import { observeClient, ObserveDaemonDownError, type HubEntryFilter, type InsightsFilter, type PageQuery, type ScoreListFilter, type SessionListFilter, type TraceListFilter } from './observeClient'; +import { observeClient, ObserveDaemonDownError, type EvaluatorListFilter, type HubEntryFilter, type InsightsFilter, type PageQuery, type ScoreListFilter, type SessionListFilter, type TraceListFilter } from './observeClient'; import type { ObserveAgentRow, ObserveDatasetDetail, ObserveDatasetList, + ObserveEvaluator, + ObserveEvaluatorList, ObserveQueue, ObserveQueueItem, ObserveQueueList, @@ -211,3 +213,18 @@ export function useQueue(queueId: string, refreshKey?: unknown): ObserveQuery { return useObserveQuery(() => observeClient.nextQueueItem(queueId), [queueId, refreshKey]); } + +/** `GET /api/v1/evaluators` - a page of online evaluator rules. */ +export function useEvaluators(filter: EvaluatorListFilter, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listEvaluators(filter), [JSON.stringify(filter), refreshKey]); +} + +/** `GET /api/v1/evaluators/:id` - one rule; `data` is `undefined` for an unknown id. */ +export function useEvaluator(ruleId: string, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.evaluator(ruleId), [ruleId, refreshKey]); +} + +/** The newest `limit` online-evaluator scores (`GET /api/v1/scores?source=online`), newest first. */ +export function useOnlineScores(limit: number, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listScores({ source: 'online', limit }), [limit, refreshKey]); +} diff --git a/apps/agent-forge/src/screens/Placeholder.tsx b/apps/agent-forge/src/screens/Placeholder.tsx deleted file mode 100644 index abed0cd1b..000000000 --- a/apps/agent-forge/src/screens/Placeholder.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import type { ReactNode } from 'react'; - -/** Stand-in for a screen that has a nav item but no content yet: what it is, what's coming, and a way back; `nav` is an optional sub-navigation above it. */ -export function Placeholder({ title, blurb, nav }: { title: string; blurb: string; nav?: ReactNode }) { - return ( -
- {nav} -

{title}

-

{blurb}

- Back to Agents -
- ); -} diff --git a/apps/agent-forge/src/screens/__tests__/routes.test.tsx b/apps/agent-forge/src/screens/__tests__/routes.test.tsx index c416542fe..3124dc6f8 100644 --- a/apps/agent-forge/src/screens/__tests__/routes.test.tsx +++ b/apps/agent-forge/src/screens/__tests__/routes.test.tsx @@ -21,7 +21,6 @@ vi.mock('../../state/AppState', () => ({ useAppState: () => appState })); import { screenFor, useScreen } from '../routes'; import { ROUTES } from '../routeTable'; -import { Placeholder } from '../Placeholder'; import { EvalsTabs } from '../evals/EvalsTabs'; import { Topbar } from '../../components/Topbar'; @@ -66,7 +65,8 @@ describe('screenFor', () => { expect(screenFor('#/evals/scores')).toBe('scores'); expect(screenFor('#/evals/datasets')).toBe('datasets'); expect(screenFor('#/evals/datasets/golden.v1?offset=50')).toBe('datasets'); - expect(screenFor('#/evals/evaluators')).toBe('evals'); + expect(screenFor('#/evals/evaluators')).toBe('evaluators'); + expect(screenFor('#/evals/evaluators/r1?trace=t1')).toBe('evaluators'); expect(screenFor('#/evals/queues')).toBe('queues'); expect(screenFor('#/evals/queues/q1')).toBe('queues'); expect(screenFor('#/playground')).toBe('playground'); @@ -79,6 +79,9 @@ describe('screenFor', () => { expect(screenFor('')).toBeNull(); expect(screenFor('#/')).toBeNull(); expect(screenFor('#/nope')).toBeNull(); + // Every Evaluations section has its own screen now; an unknown one is an unknown route. + expect(screenFor('#/evals/nope')).toBeNull(); + expect(screenFor('#/evals/nope/1')).toBeNull(); }); it("gives every route an example path that lands on that route's screen", () => { for (const { pattern, screen, example } of ROUTES) { @@ -171,15 +174,6 @@ describe('Topbar', () => { }); }); -describe('Placeholder', () => { - it('shows a title, what is coming, and a link back to Agents', () => { - render(); - expect(container.querySelector('h2')?.textContent).toBe('Playground'); - expect(container.textContent).toContain('Coming soon.'); - expect(container.querySelector('a[href="#/agents"]')).not.toBeNull(); - }); -}); - describe('EvalsTabs', () => { it('links every Evaluations screen and marks the current one', () => { window.history.replaceState(null, '', '/?token=secret#/evals/datasets/golden'); diff --git a/apps/agent-forge/src/screens/evals/evals.css b/apps/agent-forge/src/screens/evals/evals.css index 8b5fc78e6..4acb5542d 100644 --- a/apps/agent-forge/src/screens/evals/evals.css +++ b/apps/agent-forge/src/screens/evals/evals.css @@ -5,10 +5,7 @@ gap: 4px; border-bottom: 1px solid var(--border); } -.screen-placeholder .evals-tabs { - margin-bottom: 16px; -} -/* Two classes: outranks a screen's own `a` colour (the placeholder's accent links). */ +/* Two classes: outranks a screen's own `a` colour. */ .evals-tabs .evals-tab { padding: 6px 10px; border-bottom: 2px solid transparent; diff --git a/apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx b/apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx new file mode 100644 index 000000000..396836a04 --- /dev/null +++ b/apps/agent-forge/src/screens/evaluators/EvaluatorDetail.tsx @@ -0,0 +1,152 @@ +/** + * `#/evals/evaluators/:id` - one online evaluator rule: the shared form (a + * save sends only the changed fields, `PUT /api/v1/evaluators/:id`), Delete + * (after a confirm), and "Test on trace" - a dry run of the rule on one trace + * through the Studio server (`POST /observe/api/v1/evaluators/:id/test`, + * which runs a judge with Studio's provider keys and forwards a code rule to + * the daemon), showing the value and comment; no score is written. + * `?trace=` prefills the trace id. + */ +import { useState, type FormEvent } from 'react'; +import { navigate } from '../../router/router'; +import { useRoute } from '../../router/useRoute'; +import { ObserveDaemonDown, ObserveError } from '../../observe/chrome'; +import { ObserveApiError, observeClient } from '../../observe/observeClient'; +import type { NewEvaluator, ObserveEvaluator, ObserveEvaluatorTestResult } from '../../observe/types'; +import { useAgents, useDaemonDown, useEvaluator } from '../../observe/useObserve'; +import { EmptyState } from '../../ui/EmptyState'; +import { useToast } from '../../ui/Toast'; +import { useProviderKeys } from '../playground/ModelPicker'; +import { draftFrom, evaluatorPatch } from './evaluatorDraft'; +import { EvaluatorForm } from './EvaluatorForm'; +import { evaluatorsPath, parseListState } from './evaluatorsUrl'; + +/** "Test on trace": a trace id, a Test button, and the dry run's value and comment. */ +function TestOnTrace({ rule, prefill }: { rule: ObserveEvaluator; prefill: string }) { + const toast = useToast(); + const [traceId, setTraceId] = useState(prefill); + const [result, setResult] = useState(); + const [notFound, setNotFound] = useState(false); + const [running, setRunning] = useState(false); + + const run = async (e: FormEvent) => { + e.preventDefault(); + const id = traceId.trim(); + if (!id) return; + setRunning(true); + setResult(undefined); + setNotFound(false); + try { + setResult(await observeClient.testEvaluator(rule.rule_id, id)); + } catch (failure) { + if (failure instanceof ObserveApiError && failure.status === 404) setNotFound(true); + else toast.show({ tone: 'error', message: `Test failed: ${(failure as Error).message}` }); + } finally { + setRunning(false); + } + }; + + return ( +
void run(e)}> +

Test on trace

+

Runs this rule once on one trace. No score is written.

+
+ + +
+ {notFound && ( +

+ Trace not found +

+ )} + {result && ( +
+
Value
+
{result.value === null ? 'null' : String(result.value)}
+
Comment
+
{result.comment ?? '-'}
+
+ )} +
+ ); +} + +/** The edit form, Delete and Test on trace for a loaded rule; a save or delete reports through a toast. */ +function EditRule({ rule, back, onSaved }: { rule: ObserveEvaluator; back: string; onSaved: () => void }) { + const toast = useToast(); + const keys = useProviderKeys(); + const agents = useAgents(); + const { query } = useRoute(); + + const save = async (body: NewEvaluator) => { + const patch = evaluatorPatch(rule, body); + if (Object.keys(patch).length === 0) { + toast.show({ message: 'No changes to save.' }); + return false; + } + await observeClient.updateEvaluator(rule.rule_id, patch); + toast.show({ tone: 'success', message: `Saved evaluator ${body.name}.` }); + onSaved(); + return false; + }; + const remove = async () => { + if (!window.confirm(`Delete evaluator ${rule.name}? Its past scores stay.`)) return; + try { + await observeClient.deleteEvaluator(rule.rule_id); + toast.show({ tone: 'success', message: `Deleted evaluator ${rule.name}.` }); + navigate(back); + } catch (failure) { + toast.show({ tone: 'error', message: `Could not delete ${rule.name}: ${(failure as Error).message}` }); + } + }; + + return ( + <> + void remove()}> + Delete + + } + /> + + + ); +} + +/** `#/evals/evaluators/:id`; an unknown rule shows an empty state with the way back. */ +export function EvaluatorDetail({ ruleId }: { ruleId: string }) { + const { query } = useRoute(); + const [refreshKey, setRefreshKey] = useState(0); + const rule = useEvaluator(ruleId, refreshKey); + const back = evaluatorsPath(parseListState(query)); + const backLink = All evaluators; + if (useDaemonDown(false, [rule])) return ; + let body; + if (rule.loading && !rule.data) body =

Loading evaluator...

; + else if (rule.error) body = ; + else if (!rule.data) body = ; + else body = setRefreshKey((k) => k + 1)} />; + return ( + <> + +

Evaluator {rule.data ? {rule.data.name} : null}

+ {body} + + ); +} diff --git a/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx new file mode 100644 index 000000000..befcab22f --- /dev/null +++ b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx @@ -0,0 +1,234 @@ +/** + * The evaluator form the list's "New evaluator" and the edit view share: + * name, kind (`code` with a function textarea, or `judge` with a model and a + * rubric), the trace filter (agent, status, name contains), the sample rate + * and the enabled flag. A judge needs a provider key: without one in Studio's + * settings the judge option is disabled, with a hint that links Settings. + * A failed save (a taken name is the daemon's 409) shows inline. + */ +import { useState, type FormEvent, type ReactNode } from 'react'; +import type { ProviderKeyStatus } from '../../../shared/wireTypes'; +import { ObserveApiError } from '../../observe/observeClient'; +import type { NewEvaluator, ObserveAgentRow } from '../../observe/types'; +import { MOCK_PROVIDER, defaultProvider, providerOptions } from '../playground/request'; +import { draftProblem, toNewEvaluator, type EvaluatorDraft } from './evaluatorDraft'; + +type SetField = (field: K, value: EvaluatorDraft[K]) => void; + +/** The save failure shown inline: a 409 is a taken name, anything else its message. */ +const saveError = (error: unknown): string => + error instanceof ObserveApiError && error.status === 409 + ? 'Could not save: an evaluator with that name exists.' + : `Could not save the evaluator: ${(error as Error).message}`; + +function KindField({ + draft, + choose, + judgeAllowed, +}: { + draft: EvaluatorDraft; + choose: (kind: EvaluatorDraft['kind']) => void; + judgeAllowed: boolean; +}) { + return ( +
+ Kind + + + {!judgeAllowed && ( +

+ Add a provider key in Settings to use judge evaluators. +

+ )} +
+ ); +} + +function FilterFields({ draft, set, agents }: { draft: EvaluatorDraft; set: SetField; agents: ObserveAgentRow[] }) { + const known = agents.some((a) => a.agent_id === draft.agentId); + return ( +
+ Runs on traces where + + + +
+ ); +} + +function JudgeFields({ draft, set, keys }: { draft: EvaluatorDraft; set: SetField; keys: readonly ProviderKeyStatus[] }) { + const options = providerOptions(keys); + if (draft.provider !== '' && !options.includes(draft.provider)) options.unshift(draft.provider); + const isMock = draft.provider === MOCK_PROVIDER; + return ( + <> +
+ + +
+