From e5e0437fdd028521aab034372b684f0e7cbfefcd Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 25 Oct 2026 01:53:46 +0300 Subject: [PATCH 1/6] feat(observe): filter the trace list by prompt (prompt=[@]) Co-Authored-By: Claude Opus 5.5 --- docs/observability.md | 4 +++- src/observe/routes/index.test.ts | 8 +++++-- src/observe/routes/traces.ts | 16 +++++++++++++ src/observe/store.test.ts | 40 ++++++++++++++++++++++++++++++++ src/observe/store.ts | 17 +++++++++++++- 5 files changed, 81 insertions(+), 4 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index db215cbcb..e044a9c4d 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -630,11 +630,13 @@ trace, newest first. Query parameters (all optional): | `q` | Case-insensitive (ASCII) substring of the root observation's name, input or output. `%` and `_` match literally. | | `min_duration_ms` | Traces at least this long (`end - start`). | | `model` | Traces where any observation used this model. | +| `prompt` | `` or `@`: traces where any observation ran that registry prompt (linked by a `prompt_use` event), any version or just that one. | | `sort` | `start` (default), `duration`, `tokens` (input + output) or `cost`. Ties break on `trace_id`. | | `order` | `desc` (default) or `asc`. | | `limit`, `offset` | Page size (1-500, default 50) and offset. | -An unknown `sort` or `order`, or a non-numeric `min_duration_ms`, answers +An unknown `sort` or `order`, a non-numeric `min_duration_ms`, or a `prompt` +with an empty name or a non-integer version answers `400` with code `LOUSHO_REQUEST_INVALID`. Each row has `trace_id`, `name`, `agent_id`, `session_id`, `start`, `end`, diff --git a/src/observe/routes/index.test.ts b/src/observe/routes/index.test.ts index eb38a5b55..c8de51ea7 100644 --- a/src/observe/routes/index.test.ts +++ b/src/observe/routes/index.test.ts @@ -61,12 +61,16 @@ describe('GET /api/v1/traces query params', () => { }; it('accepts the filter, search and sort params', () => { - const { status, body } = call('from=1&to=2&q=ord_5&min_duration_ms=10&model=m&sort=duration&order=asc'); + const { status, body } = call('from=1&to=2&q=ord_5&min_duration_ms=10&model=m&sort=duration&order=asc&prompt=mira@2'); expect(status).toBe(200); expect(body).toMatchObject({ traces: [], total: 0 }); }); - it.each(['sort=bogus', 'order=sideways', 'min_duration_ms=fast'])('rejects %s with 400 LOUSHO_REQUEST_INVALID', (search) => { + it('accepts prompt= without a version', () => { + expect(call('prompt=mira').status).toBe(200); + }); + + it.each(['sort=bogus', 'order=sideways', 'min_duration_ms=fast', 'prompt=', 'prompt=mira@', 'prompt=mira@two', 'prompt=@2', 'prompt=mira@1.5'])('rejects %s with 400 LOUSHO_REQUEST_INVALID', (search) => { const { status, body } = call(search); expect(status).toBe(400); expect(body.code).toBe('LOUSHO_REQUEST_INVALID'); diff --git a/src/observe/routes/traces.ts b/src/observe/routes/traces.ts index 91157552f..f0c97571a 100644 --- a/src/observe/routes/traces.ts +++ b/src/observe/routes/traces.ts @@ -29,6 +29,19 @@ function listShape(query: URLSearchParams): Pick` or `prompt=@`: absent, the parsed filter, or the message for the 400 a bad value earns. */ +function promptParam(query: URLSearchParams): TraceFilter['prompt'] | string { + const raw = query.get('prompt'); + if (raw === null) return undefined; + const at = raw.lastIndexOf('@'); + const name = at === -1 ? raw : raw.slice(0, at); + const version = at === -1 ? undefined : raw.slice(at + 1); + if (name.trim() === '' || (version !== undefined && !/^\d+$/.test(version))) { + return `Invalid prompt '${raw}': expected or @ (an integer).`; + } + return version === undefined ? { name } : { name, version: Number(version) }; +} + export const tracesRoutes: readonly Route[] = [ { method: 'GET', @@ -36,6 +49,8 @@ export const tracesRoutes: readonly Route[] = [ handle: ({ db, res, query }) => { const shape = listShape(query); if (typeof shape === 'string') return badRequest(res, shape); + const prompt = promptParam(query); + if (typeof prompt === 'string') return badRequest(res, prompt); const filter: TraceFilter = { agent_id: query.get('agent_id') ?? undefined, project: query.get('project') ?? undefined, @@ -47,6 +62,7 @@ export const tracesRoutes: readonly Route[] = [ to: timeParam(query, 'to'), q: query.get('q') ?? undefined, model: query.get('model') ?? undefined, + prompt, ...shape, limit: intParam(query, 'limit', 50, LIST_LIMIT), offset: intParam(query, 'offset', 0, { min: 0, max: Number.MAX_SAFE_INTEGER }), diff --git a/src/observe/store.test.ts b/src/observe/store.test.ts index b91f87e15..e03b02c89 100644 --- a/src/observe/store.test.ts +++ b/src/observe/store.test.ts @@ -874,3 +874,43 @@ describe('insights math and per-agent series (O14)', () => { } }); }); + +describe('trace list prompt filter', () => { + // t1 ran mira@1, t2 ran mira@2 (on a child generation), t3 ran other@1, t4 ran no prompt + const seed = (database: SqlDatabase): void => { + const insert = database.prepare( + 'INSERT INTO prompts (prompt_id, name, version, labels, text, variables, hash, created_at) VALUES (?, ?, ?, NULL, ?, NULL, NULL, ?)' + ); + insert.run('p-mira-1', 'mira', 1, 'v1', T0); + insert.run('p-mira-2', 'mira', 2, 'v2', T0); + insert.run('p-other-1', 'other', 1, 'o1', T0); + ingestBatch(database, [ + obs({ obs_id: 'r1', trace_id: 't1', start: T0 }), + obs({ obs_id: 'r2', trace_id: 't2', start: T0 + 1000 }), + obs({ obs_id: 'g2', trace_id: 't2', parent_obs_id: 'r2', kind: 'generation', start: T0 + 1001 }), + obs({ obs_id: 'r3', trace_id: 't3', start: T0 + 2000 }), + obs({ obs_id: 'r4', trace_id: 't4', start: T0 + 3000 }), + { type: 'prompt_use', data: { obs_id: 'r1', prompt_id: 'p-mira-1', version: 1 } }, + { type: 'prompt_use', data: { obs_id: 'g2', prompt_id: 'p-mira-2', version: 2 } }, + { type: 'prompt_use', data: { obs_id: 'r3', prompt_id: 'p-other-1', version: 1 } }, + ]); + }; + const list = (database: SqlDatabase, prompt: { name: string; version?: number }) => + listTraces(database, { prompt, limit: 50, offset: 0 }); + + it('keeps traces with any observation linked to the prompt, any version or one version', () => { + const database = db(); + try { + seed(database); + expect(list(database, { name: 'mira' }).traces.map((t) => t.trace_id)).toEqual(['t2', 't1']); + expect(list(database, { name: 'mira' }).total).toBe(2); + expect(list(database, { name: 'mira', version: 2 }).traces.map((t) => t.trace_id)).toEqual(['t2']); + expect(list(database, { name: 'mira', version: 2 }).total).toBe(1); + expect(list(database, { name: 'other' }).traces.map((t) => t.trace_id)).toEqual(['t3']); + expect(list(database, { name: 'mira', version: 9 }).total).toBe(0); + expect(list(database, { name: 'nope' }).total).toBe(0); + } finally { + database.close(); + } + }); +}); diff --git a/src/observe/store.ts b/src/observe/store.ts index 28e59382a..072ee16c1 100644 --- a/src/observe/store.ts +++ b/src/observe/store.ts @@ -675,6 +675,8 @@ export interface TraceFilter { min_duration_ms?: number; /** Traces where any observation used this model. */ model?: string; + /** Traces where any observation is linked (`prompt_uses`) to this registry prompt - any version, or just `version`. */ + prompt?: { name: string; version?: number }; /** Default `start`. Ties break on `trace_id` in the same direction. */ sort?: TraceSort; /** Default `desc`. */ @@ -754,8 +756,20 @@ const ROOT_OF = `observations r WHERE r.trace_id = o.trace_id AND r.parent_obs_i /** `value` as a LIKE pattern matching it as a literal substring (paired with `ESCAPE '\'`). */ const likeContains = (value: string): string => `%${value.replace(/[\\%_]/g, (c) => `\\${c}`)}%`; +/** `AND EXISTS` an observation of the trace linked to the prompt `name` (at `version`, when given). */ +function promptUseExists(prompt: { name: string; version?: number }, arg: SqlArg): string { + // `arg` binds positionally, so call it in SQL text order: name, then version. + const name = arg(prompt.name); + const version = prompt.version === undefined ? '' : ` AND p.version = ${arg(prompt.version)}`; + return ( + ` AND EXISTS (SELECT 1 FROM observations u JOIN prompt_uses pu ON pu.obs_id = u.obs_id` + + ` JOIN prompts p ON p.prompt_id = pu.prompt_id` + + ` WHERE u.trace_id = o.trace_id AND p.name = ${name}${version})` + ); +} + /** - * The window/duration/model/search filters. They judge the whole trace, so + * The window/duration/model/prompt/search filters. They judge the whole trace, so * they sit in HAVING; `from`/`to` bound the trace start, half-open. */ function traceHavingExtras(filter: TraceFilter, arg: SqlArg): string { @@ -768,6 +782,7 @@ function traceHavingExtras(filter: TraceFilter, arg: SqlArg): string { if (filter.model !== undefined) { having += ` AND EXISTS (SELECT 1 FROM observations m WHERE m.trace_id = o.trace_id AND m.model = ${arg(filter.model)})`; } + if (filter.prompt !== undefined) having += promptUseExists(filter.prompt, arg); if (filter.q) { const like = likeContains(filter.q); const col = (name: string): string => `r.${name} LIKE ${arg(like)} ESCAPE '\\'`; From e8a94d9a418c9f82bef46067da6c025b30e6129e Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 25 Oct 2026 01:54:41 +0300 Subject: [PATCH 2/6] feat(studio): observe proxy forwards an explicit write allowlist (prompt label move) Co-Authored-By: Claude Opus 5.5 --- .../server/__tests__/observeProxy.test.ts | 42 +++++++++++++-- apps/agent-forge/server/observeProxy.ts | 53 +++++++++++++------ 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/apps/agent-forge/server/__tests__/observeProxy.test.ts b/apps/agent-forge/server/__tests__/observeProxy.test.ts index 69d855773..881b0f826 100644 --- a/apps/agent-forge/server/__tests__/observeProxy.test.ts +++ b/apps/agent-forge/server/__tests__/observeProxy.test.ts @@ -1,9 +1,9 @@ /** * `/observe/**` -> the observe daemon: the studio server forwards the * panels' GETs same-origin, because the daemon itself serves no CORS - * headers. A dead daemon answers 502 with the `lousho observe start` hint; - * writes are forwarded only for the allowlist (feedback scores), every other - * non-GET stays reporter-only (405). + * headers. A dead daemon answers 502 with the `lousho observe start` hint. + * Writes go through only when OBSERVE_WRITE_ALLOWLIST names the method and + * path (feedback scores, a prompt label move); every other non-GET stays 405. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as http from 'node:http'; @@ -13,6 +13,7 @@ import * as os from 'node:os'; import { once } from 'node:events'; import request from 'supertest'; import { createApp } from '../app'; +import { OBSERVE_WRITE_ALLOWLIST } from '../observeProxy'; import { RunManager } from '../runRegistry'; import { FileCheckpointStore } from '../checkpointStore'; import { FileApprovalStore } from '../approvalStore'; @@ -37,7 +38,7 @@ describe('observe proxy', () => { }); app = createApp({ agentStore, runManager, baseDir }); - // A stand-in for `lousho observe`: echoes the path it was asked for. + // A stand-in for `lousho observe`: echoes the method, path, content-type and body it was sent. daemon = http.createServer((req, res) => { let body = ''; req.on('data', (chunk: Buffer) => (body += chunk.toString())); @@ -99,4 +100,37 @@ describe('observe proxy', () => { expect((await request(app).put('/observe/api/v1/scores').send({})).status).toBe(405); expect((await request(app).delete('/observe/api/v1/scores')).status).toBe(405); }); + + it('forwards an allowlisted PUT (prompt label move) with its JSON body and content-type', async () => { + const res = await request(app).put('/observe/api/v1/prompts/mira/labels/prod').send({ version: 2 }); + expect(res.status).toBe(200); + expect(res.body.method).toBe('PUT'); + expect(res.body.echoed).toBe('/api/v1/prompts/mira/labels/prod'); + expect(res.body.contentType).toContain('application/json'); + expect(res.body.body).toEqual({ version: 2 }); + }); + + it.each([ + ['post', '/observe/api/v1/prompts'], + ['put', '/observe/api/v1/prompts/mira'], + ['put', '/observe/api/v1/prompts/mira/labels/prod/extra'], + ['delete', '/observe/api/v1/prompts/mira/labels/prod'], + ['post', '/observe/api/v1/prompts/mira/labels/prod'], + ['post', '/observe/v1/traces'], + ] as const)('keeps %s %s at 405', async (method, url) => { + const res = await request(app)[method](url).send({ version: 1 }); + expect(res.status).toBe(405); + }); + + it('rejects an allowlisted write whose body is over 1 MiB with 413', async () => { + const res = await request(app) + .put('/observe/api/v1/prompts/mira/labels/prod') + .send({ version: 1, pad: 'x'.repeat(1024 * 1024) }); + expect(res.status).toBe(413); + }); + + it('exports one write allowlist of method + path pairs: feedback scores and the label move', () => { + expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT']); + expect(OBSERVE_WRITE_ALLOWLIST.every((entry) => entry.path instanceof RegExp)).toBe(true); + }); }); diff --git a/apps/agent-forge/server/observeProxy.ts b/apps/agent-forge/server/observeProxy.ts index 4fe184fbe..31dfc3556 100644 --- a/apps/agent-forge/server/observeProxy.ts +++ b/apps/agent-forge/server/observeProxy.ts @@ -8,24 +8,40 @@ * the file. `LOUSHO_OBSERVE_URL` repoints the upstream (another port, or a * hosted collector implementing the same ingest/read API). * - * GET is forwarded for every path; writes only for the explicit - * {@link WRITE_ROUTES} allowlist (today: human feedback scores), with the - * JSON body app.ts's `express.json` (2 MB cap) already parsed. Everything - * else non-GET - the ingest endpoint above all - answers 405 and stays - * reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it carries the - * same per-launch token requirement as the rest of the studio API. + * GET is forwarded for every path. Writes are forwarded only when + * {@link OBSERVE_WRITE_ALLOWLIST} names the method and path - a short list of + * Studio actions (human feedback scores, moving a prompt label) - with the + * JSON body app.ts's `express.json` already parsed, re-serialised and capped + * at 1 MiB. Every other non-GET - the ingest endpoint above all - answers 405 + * and stays reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it + * carries the same per-launch token requirement as the rest of the studio API. */ -import type { Express } from 'express'; +import type { Express, Request } from 'express'; const DEFAULT_OBSERVE_URL = 'http://127.0.0.1:7777'; const PROXY_PREFIX = '/observe'; const UPSTREAM_TIMEOUT_MS = 8000; +const MAX_WRITE_BODY_BYTES = 1024 * 1024; -/** The daemon writes Studio may forward (method + path below `/observe`). Add one entry per new write surface. */ -const WRITE_ROUTES: { method: 'POST' | 'PUT'; path: RegExp }[] = [{ method: 'POST', path: /^\/api\/v1\/scores$/ }]; +/** A daemon write the proxy forwards: the method plus a regex over the path after `/observe` (no query). */ +export interface ObserveWrite { + method: 'POST' | 'PUT'; + path: RegExp; +} + +/** The only non-GET requests the proxy forwards; a later Studio write is one more line here. */ +export const OBSERVE_WRITE_ALLOWLIST: readonly ObserveWrite[] = [ + { method: 'POST', path: /^\/api\/v1\/scores$/ }, + { method: 'PUT', path: /^\/api\/v1\/prompts\/[^/]+\/labels\/[^/]+$/ }, +]; + +const isAllowedWrite = (req: Request): boolean => + OBSERVE_WRITE_ALLOWLIST.some((entry) => entry.method === req.method && entry.path.test(req.path)); -function isAllowedWrite(method: string, path: string): boolean { - return WRITE_ROUTES.some((route) => route.method === method && route.path.test(path)); +/** The JSON body to forward (express.json() already parsed it), or `undefined` when it is over the cap. */ +function writeBody(req: Request): string | undefined { + const body = JSON.stringify(req.body ?? {}); + return Buffer.byteLength(body, 'utf8') > MAX_WRITE_BODY_BYTES ? undefined : body; } /** The daemon the Studio reads (and playground runs report to): `LOUSHO_OBSERVE_URL`, default loopback :7777. */ @@ -35,9 +51,14 @@ export function observeTarget(): string { export function registerObserveProxy(app: Express): void { app.use(PROXY_PREFIX, async (req, res) => { - const write = req.method !== 'GET'; - if (write && !isAllowedWrite(req.method, req.path)) { - res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted writes.' }); + const isWrite = req.method !== 'GET'; + if (isWrite && !isAllowedWrite(req)) { + res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted Studio writes.' }); + return; + } + const body = isWrite ? writeBody(req) : undefined; + if (isWrite && body === undefined) { + res.status(413).json({ error: 'Observe write bodies are capped at 1 MiB.' }); return; } // Which daemon this proxy points at, so the down state can name the right LOUSHO_OBSERVE value. @@ -50,8 +71,8 @@ export function registerObserveProxy(app: Express): void { try { response = await fetch(upstream, { method: req.method, - headers: write ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' }, - body: write ? JSON.stringify(req.body ?? {}) : undefined, + headers: isWrite ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' }, + body, signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), }); } catch { From f45e9e434f77b5a3a1a67c39269e8dd359d2c3e9 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 25 Oct 2026 02:05:48 +0300 Subject: [PATCH 3/6] docs: regenerate llms-full for the trace list prompt filter Co-Authored-By: Claude Opus 5.5 --- llms-full.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llms-full.txt b/llms-full.txt index d275bfe2f..7a722db9c 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -14434,11 +14434,13 @@ trace, newest first. Query parameters (all optional): | `q` | Case-insensitive (ASCII) substring of the root observation's name, input or output. `%` and `_` match literally. | | `min_duration_ms` | Traces at least this long (`end - start`). | | `model` | Traces where any observation used this model. | +| `prompt` | `` or `@`: traces where any observation ran that registry prompt (linked by a `prompt_use` event), any version or just that one. | | `sort` | `start` (default), `duration`, `tokens` (input + output) or `cost`. Ties break on `trace_id`. | | `order` | `desc` (default) or `asc`. | | `limit`, `offset` | Page size (1-500, default 50) and offset. | -An unknown `sort` or `order`, or a non-numeric `min_duration_ms`, answers +An unknown `sort` or `order`, a non-numeric `min_duration_ms`, or a `prompt` +with an empty name or a non-integer version answers `400` with code `LOUSHO_REQUEST_INVALID`. Each row has `trace_id`, `name`, `agent_id`, `session_id`, `start`, `end`, From 26ac4973a13c69de00fac59d2e09949049bfdf9d Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 25 Oct 2026 02:05:49 +0300 Subject: [PATCH 4/6] feat(studio): prompts screen Co-Authored-By: Claude Opus 5.5 --- apps/agent-forge/e2e/a11y.spec.ts | 2 +- .../src/__tests__/contrast.test.ts | 2 + .../observe/__tests__/observeClient.test.ts | 47 ++++ apps/agent-forge/src/observe/observeClient.ts | 31 ++- apps/agent-forge/src/observe/types.ts | 33 +++ apps/agent-forge/src/observe/useObserve.ts | 10 + .../src/screens/prompts/LabelChips.tsx | 13 ++ .../src/screens/prompts/PromptDetail.tsx | 215 ++++++++++++++++++ .../src/screens/prompts/PromptDiff.tsx | 74 ++++++ .../src/screens/prompts/PromptsScreen.tsx | 64 ++++++ .../prompts/__tests__/PromptsScreen.test.tsx | 190 ++++++++++++++++ .../prompts/__tests__/lineDiff.test.ts | 65 ++++++ .../src/screens/prompts/lineDiff.ts | 127 +++++++++++ .../src/screens/prompts/prompts.css | 135 +++++++++++ .../src/screens/prompts/promptsUrl.ts | 49 ++++ apps/agent-forge/src/screens/routes.tsx | 3 +- .../src/screens/traces/TracesScreen.tsx | 16 +- .../traces/__tests__/TracesScreen.test.tsx | 14 ++ .../traces/__tests__/tracesUrl.test.ts | 3 +- .../agent-forge/src/screens/traces/traces.css | 26 +++ .../src/screens/traces/tracesUrl.ts | 6 +- apps/agent-forge/src/ui/DataTable.tsx | 21 +- .../src/ui/__tests__/DataTable.test.tsx | 8 + apps/agent-forge/src/ui/dataTable.css | 5 + 24 files changed, 1148 insertions(+), 11 deletions(-) create mode 100644 apps/agent-forge/src/screens/prompts/LabelChips.tsx create mode 100644 apps/agent-forge/src/screens/prompts/PromptDetail.tsx create mode 100644 apps/agent-forge/src/screens/prompts/PromptDiff.tsx create mode 100644 apps/agent-forge/src/screens/prompts/PromptsScreen.tsx create mode 100644 apps/agent-forge/src/screens/prompts/__tests__/PromptsScreen.test.tsx create mode 100644 apps/agent-forge/src/screens/prompts/__tests__/lineDiff.test.ts create mode 100644 apps/agent-forge/src/screens/prompts/lineDiff.ts create mode 100644 apps/agent-forge/src/screens/prompts/prompts.css create mode 100644 apps/agent-forge/src/screens/prompts/promptsUrl.ts diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index 57ec699c0..4903c69d8 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -113,7 +113,7 @@ test('Ctrl+K opens the command palette; ? lists the shortcuts', async ({ page }) }); // One
, one h1, the banner and the primary nav on every screen, not just Agents. -for (const route of ['#/traces', '#/traces?status=error', '#/traces/e2e-trace?obs=e2e-obs', '#/threads', '#/threads/s1', '#/threads?status=active', '#/monitoring', '#/evals/scores', '#/settings', '#/playground']) { +for (const route of ['#/traces', '#/traces?status=error', '#/traces/e2e-trace?obs=e2e-obs', '#/threads', '#/threads/s1', '#/threads?status=active', '#/monitoring', '#/evals/scores', '#/prompts', '#/prompts/mira@2?diff=1,2', '#/settings', '#/playground']) { test(`${route} keeps the landmarks and exactly one main`, async ({ page }) => { await page.goto(`/?token=e2e-studio-token${route}`); await expect(page.locator('.screen-main')).toBeVisible(); diff --git a/apps/agent-forge/src/__tests__/contrast.test.ts b/apps/agent-forge/src/__tests__/contrast.test.ts index 3cb2d452b..c05a23614 100644 --- a/apps/agent-forge/src/__tests__/contrast.test.ts +++ b/apps/agent-forge/src/__tests__/contrast.test.ts @@ -49,6 +49,8 @@ const PAIRS: [text: string, background: string][] = [ ['text', 'surface-2'], // Monitoring: the selected window button draws text on the accent tint. ['text', 'accent-soft'], + // Prompt label chips, the selected DataTable row and the Traces prompt chip (its x in text-muted) on the accent tint. + ['text-muted', 'accent-soft'], ]; describe('theme contrast', () => { diff --git a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts index 54ed8cf0b..34b0bd6dd 100644 --- a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts +++ b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts @@ -124,4 +124,51 @@ describe('ObserveClient', () => { expect.anything() ); }); + + it('passes the prompt trace filter through', async () => { + const fetchFn = vi.fn(async () => jsonResponse(200, { traces: [], total: 0, limit: 50, offset: 0 })); + await clientWith(fetchFn as typeof fetch).listTraces({ prompt: 'mira@2' }); + expect(fetchFn).toHaveBeenCalledWith('/observe/api/v1/traces?prompt=mira%402', expect.anything()); + }); + + it('lists prompts and reads one prompt history, undefined on 404', async () => { + const fetchFn = vi.fn(async (url: string) => + url.endsWith('/prompts') + ? jsonResponse(200, { prompts: [{ name: 'mira', version: 2, labels: ['prod'], versions: 2, uses: 3 }] }) + : jsonResponse(404, { error: "No prompt 'x'.", code: 'LOUSHO_REQUEST_INVALID' }) + ); + const client = clientWith(fetchFn as unknown as typeof fetch); + expect((await client.listPrompts())[0]).toMatchObject({ name: 'mira', uses: 3 }); + expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/prompts', expect.anything()); + expect(await client.promptVersions('x y')).toBeUndefined(); + expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/prompts/x%20y', expect.anything()); + }); + + it('movePromptLabel PUTs {version} as JSON with the studio token and returns the move', async () => { + window.sessionStorage.setItem('lousho-studio-token', 'tok-123'); + try { + const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => { + void init; + return jsonResponse(200, { name: 'mira', label: 'prod', version: 2, previous_version: 1 }); + }); + const moved = await clientWith(fetchFn as typeof fetch).movePromptLabel('mira', 'prod', 2); + expect(moved).toEqual({ name: 'mira', label: 'prod', version: 2, previous_version: 1 }); + const [url, init] = fetchFn.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('/observe/api/v1/prompts/mira/labels/prod'); + expect(init.method).toBe('PUT'); + expect(JSON.parse(String(init.body))).toEqual({ version: 2 }); + const headers = new Headers(init.headers); + expect(headers.get('content-type')).toBe('application/json'); + expect(headers.get('x-lousho-studio-token')).toBe('tok-123'); + } finally { + window.sessionStorage.removeItem('lousho-studio-token'); + } + }); + + it('maps write failures like reads: 502 is daemon-down, other errors carry the daemon message', async () => { + const down = clientWith((async () => jsonResponse(502, { error: 'unreachable' })) as typeof fetch); + await expect(down.movePromptLabel('mira', 'prod', 2)).rejects.toBeInstanceOf(ObserveDaemonDownError); + const missing = clientWith((async () => jsonResponse(404, { error: "No prompt 'mira@9'." })) as typeof fetch); + await expect(missing.movePromptLabel('mira', 'prod', 9)).rejects.toMatchObject({ status: 404, message: "No prompt 'mira@9'." }); + }); }); diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts index b35c30ddf..238f5e9d7 100644 --- a/apps/agent-forge/src/observe/observeClient.ts +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -14,6 +14,9 @@ import type { ObserveAgentRow, ObserveHealth, ObserveInsights, + ObservePromptHistory, + ObservePromptLabelMove, + ObservePromptRow, ObserveScorePassRate, ObserveScoreRow, ObserveSessionDetail, @@ -71,6 +74,8 @@ interface TraceListQuery { model?: string; sort?: TraceSort; order?: 'asc' | 'desc'; + /** `` or `@`: traces where any observation ran that registry prompt. */ + prompt?: string; } export interface TraceListFilter extends TraceListQuery { @@ -153,7 +158,7 @@ export class ObserveClient { private async request( path: string, - opts: { query?: Record; method?: 'GET' | 'POST'; body?: unknown } = {} + opts: { query?: Record; method?: 'GET' | 'POST' | 'PUT'; body?: unknown } = {} ): Promise { const headers = new Headers({ accept: 'application/json' }); // The studio token only travels same-origin (to the studio server's @@ -245,6 +250,30 @@ export class ObserveClient { return this.request('/api/v1/insights', { query: { ...filter } }); } + /** `GET /api/v1/prompts` - one row per prompt name (its latest version), with version and use counts. */ + async listPrompts(): Promise { + const body = await this.request<{ prompts: ObservePromptRow[] }>('/api/v1/prompts'); + return body.prompts; + } + + /** `GET /api/v1/prompts/:name` - the version history, newest first; `undefined` on the daemon's 404. */ + async promptVersions(name: string): Promise { + try { + return await this.request(`/api/v1/prompts/${encodeURIComponent(name)}`); + } catch (error) { + if (error instanceof ObserveApiError && error.status === 404) return undefined; + throw error; + } + } + + /** `PUT /api/v1/prompts/:name/labels/:label {version}` - point `label` at `version`, moving it off its old one. */ + movePromptLabel(name: string, label: string, version: number): Promise { + return this.request(`/api/v1/prompts/${encodeURIComponent(name)}/labels/${encodeURIComponent(label)}`, { + method: 'PUT', + body: { version }, + }); + } + /** `GET /api/v1/scores` - eval/judge/human score rows. */ async listScores(filter: ScoreListFilter = {}): Promise { const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { query: { ...filter } }); diff --git a/apps/agent-forge/src/observe/types.ts b/apps/agent-forge/src/observe/types.ts index 95157efb5..f9e6b29a1 100644 --- a/apps/agent-forge/src/observe/types.ts +++ b/apps/agent-forge/src/observe/types.ts @@ -191,3 +191,36 @@ export interface ObserveInsights { agents: ObserveAgentInsights[]; series: ObserveRollupRow[]; } + +/** One version of a registry prompt (`GET /api/v1/prompts/:name` rows; `labels` = the labels pointing at it, sorted). */ +export interface ObservePromptVersion { + prompt_id: string; + name: string; + version: number; + labels: string[]; + variables: unknown; + hash: string | null; + text: string | null; + created_at: number; + /** Observations linked (`prompt_use`) to this version. */ + uses: number; +} + +/** One row of `GET /api/v1/prompts`: the latest version of a name, with its version count and run uses across versions. */ +export interface ObservePromptRow extends ObservePromptVersion { + versions: number; +} + +/** `GET /api/v1/prompts/:name`: the version history, newest first, text included. */ +export interface ObservePromptHistory { + name: string; + versions: ObservePromptVersion[]; +} + +/** `PUT /api/v1/prompts/:name/labels/:label` answer: where the label points now and where it pointed before (null = new label). */ +export interface ObservePromptLabelMove { + name: string; + label: string; + version: number; + previous_version: number | null; +} diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts index cee067f21..3eed3fd24 100644 --- a/apps/agent-forge/src/observe/useObserve.ts +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -11,6 +11,8 @@ import type { ObserveAgentRow, ObserveHealth, ObserveInsights, + ObservePromptHistory, + ObservePromptRow, ObserveScorePassRate, ObserveScoreRow, ObserveSessionDetail, @@ -162,3 +164,11 @@ export function useDaemonDown( }, [down, pollMs]); return down; } + +export function usePrompts(refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listPrompts(), [refreshKey]); +} + +export function usePromptVersions(name: string, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.promptVersions(name), [name, refreshKey]); +} diff --git a/apps/agent-forge/src/screens/prompts/LabelChips.tsx b/apps/agent-forge/src/screens/prompts/LabelChips.tsx new file mode 100644 index 000000000..e6c13eb65 --- /dev/null +++ b/apps/agent-forge/src/screens/prompts/LabelChips.tsx @@ -0,0 +1,13 @@ +/** A prompt version's labels as chips; nothing when it has none. */ +export function LabelChips({ labels }: { labels: readonly string[] }) { + if (labels.length === 0) return null; + return ( +
    + {labels.map((label) => ( +
  • + {label} +
    • + ))} +
+ ); +} diff --git a/apps/agent-forge/src/screens/prompts/PromptDetail.tsx b/apps/agent-forge/src/screens/prompts/PromptDetail.tsx new file mode 100644 index 000000000..1eae9de59 --- /dev/null +++ b/apps/agent-forge/src/screens/prompts/PromptDetail.tsx @@ -0,0 +1,215 @@ +/** + * `#/prompts/:name[@ref]` - one prompt: the selected version (its text, + * labels, "Move