diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index 57ec699c0..4903c69d8 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -113,7 +113,7 @@ test('Ctrl+K opens the command palette; ? lists the shortcuts', async ({ page }) }); // One
, one h1, the banner and the primary nav on every screen, not just Agents. -for (const route of ['#/traces', '#/traces?status=error', '#/traces/e2e-trace?obs=e2e-obs', '#/threads', '#/threads/s1', '#/threads?status=active', '#/monitoring', '#/evals/scores', '#/settings', '#/playground']) { +for (const route of ['#/traces', '#/traces?status=error', '#/traces/e2e-trace?obs=e2e-obs', '#/threads', '#/threads/s1', '#/threads?status=active', '#/monitoring', '#/evals/scores', '#/prompts', '#/prompts/mira@2?diff=1,2', '#/settings', '#/playground']) { test(`${route} keeps the landmarks and exactly one main`, async ({ page }) => { await page.goto(`/?token=e2e-studio-token${route}`); await expect(page.locator('.screen-main')).toBeVisible(); diff --git a/apps/agent-forge/server/__tests__/observeProxy.test.ts b/apps/agent-forge/server/__tests__/observeProxy.test.ts index 69d855773..421bb152b 100644 --- a/apps/agent-forge/server/__tests__/observeProxy.test.ts +++ b/apps/agent-forge/server/__tests__/observeProxy.test.ts @@ -1,9 +1,9 @@ /** * `/observe/**` -> the observe daemon: the studio server forwards the * panels' GETs same-origin, because the daemon itself serves no CORS - * headers. A dead daemon answers 502 with the `lousho observe start` hint; - * writes are forwarded only for the allowlist (feedback scores), every other - * non-GET stays reporter-only (405). + * headers. A dead daemon answers 502 with the `lousho observe start` hint. + * Writes go through only when OBSERVE_WRITE_ALLOWLIST names the method and + * path (feedback scores, a prompt label move); every other non-GET stays 405. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as http from 'node:http'; @@ -13,6 +13,7 @@ import * as os from 'node:os'; import { once } from 'node:events'; import request from 'supertest'; import { createApp } from '../app'; +import { OBSERVE_WRITE_ALLOWLIST } from '../observeProxy'; import { RunManager } from '../runRegistry'; import { FileCheckpointStore } from '../checkpointStore'; import { FileApprovalStore } from '../approvalStore'; @@ -37,7 +38,7 @@ describe('observe proxy', () => { }); app = createApp({ agentStore, runManager, baseDir }); - // A stand-in for `lousho observe`: echoes the path it was asked for. + // A stand-in for `lousho observe`: echoes the method, path, content-type and body it was sent. daemon = http.createServer((req, res) => { let body = ''; req.on('data', (chunk: Buffer) => (body += chunk.toString())); @@ -99,4 +100,49 @@ describe('observe proxy', () => { expect((await request(app).put('/observe/api/v1/scores').send({})).status).toBe(405); expect((await request(app).delete('/observe/api/v1/scores')).status).toBe(405); }); + + it('forwards an allowlisted PUT (prompt label move) with its JSON body and content-type', async () => { + const res = await request(app).put('/observe/api/v1/prompts/mira/labels/prod').send({ version: 2 }); + expect(res.status).toBe(200); + expect(res.body.method).toBe('PUT'); + expect(res.body.echoed).toBe('/api/v1/prompts/mira/labels/prod'); + expect(res.body.contentType).toContain('application/json'); + expect(res.body.body).toEqual({ version: 2 }); + }); + + it.each([ + ['post', '/observe/api/v1/prompts'], + ['put', '/observe/api/v1/prompts/mira'], + ['put', '/observe/api/v1/prompts/mira/labels/prod/extra'], + ['delete', '/observe/api/v1/prompts/mira/labels/prod'], + ['post', '/observe/api/v1/prompts/mira/labels/prod'], + ['post', '/observe/v1/traces'], + ] as const)('keeps %s %s at 405', async (method, url) => { + const res = await request(app)[method](url).send({ version: 1 }); + expect(res.status).toBe(405); + }); + + it.each([ + '/observe/api/v1/prompts/%2e%2e/labels/x', + '/observe/api/v1/prompts/%2E/labels/x', + '/observe/api/v1/prompts/mira/labels/%2e%2E', + '/observe/api/v1/prompts/a%2Fb/labels/x', + '/observe/api/v1/prompts/a%5cb/labels/x', + '/observe/api/v1/prompts/%E0%A4%A/labels/x', + ])('keeps an allowlisted shape with a dot, encoded-slash or undecodable segment at 405: %s', async (url) => { + const res = await request(app).put(url).send({ version: 1 }); + expect(res.status).toBe(405); + }); + + it('rejects an allowlisted write whose body is over 1 MiB with 413', async () => { + const res = await request(app) + .put('/observe/api/v1/prompts/mira/labels/prod') + .send({ version: 1, pad: 'x'.repeat(1024 * 1024) }); + expect(res.status).toBe(413); + }); + + it('exports one write allowlist of method + path pairs: feedback scores and the label move', () => { + expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT']); + expect(OBSERVE_WRITE_ALLOWLIST.every((entry) => entry.path instanceof RegExp)).toBe(true); + }); }); diff --git a/apps/agent-forge/server/observeProxy.ts b/apps/agent-forge/server/observeProxy.ts index 4fe184fbe..10420eefc 100644 --- a/apps/agent-forge/server/observeProxy.ts +++ b/apps/agent-forge/server/observeProxy.ts @@ -8,24 +8,56 @@ * the file. `LOUSHO_OBSERVE_URL` repoints the upstream (another port, or a * hosted collector implementing the same ingest/read API). * - * GET is forwarded for every path; writes only for the explicit - * {@link WRITE_ROUTES} allowlist (today: human feedback scores), with the - * JSON body app.ts's `express.json` (2 MB cap) already parsed. Everything - * else non-GET - the ingest endpoint above all - answers 405 and stays - * reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it carries the - * same per-launch token requirement as the rest of the studio API. + * GET is forwarded for every path. Writes are forwarded only when + * {@link OBSERVE_WRITE_ALLOWLIST} names the method and path - a short list of + * Studio actions (human feedback scores, moving a prompt label) - with the + * JSON body app.ts's `express.json` already parsed, re-serialised and capped + * at 1 MiB. Every other non-GET - the ingest endpoint above all - answers 405 + * and stays reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it + * carries the same per-launch token requirement as the rest of the studio API. */ -import type { Express } from 'express'; +import type { Express, Request } from 'express'; const DEFAULT_OBSERVE_URL = 'http://127.0.0.1:7777'; const PROXY_PREFIX = '/observe'; const UPSTREAM_TIMEOUT_MS = 8000; +const MAX_WRITE_BODY_BYTES = 1024 * 1024; -/** The daemon writes Studio may forward (method + path below `/observe`). Add one entry per new write surface. */ -const WRITE_ROUTES: { method: 'POST' | 'PUT'; path: RegExp }[] = [{ method: 'POST', path: /^\/api\/v1\/scores$/ }]; +/** A daemon write the proxy forwards: the method plus a regex over the path after `/observe` (no query). */ +export interface ObserveWrite { + method: 'POST' | 'PUT'; + path: RegExp; +} + +/** The only non-GET requests the proxy forwards; a later Studio write is one more line here. */ +export const OBSERVE_WRITE_ALLOWLIST: readonly ObserveWrite[] = [ + { method: 'POST', path: /^\/api\/v1\/scores$/ }, + { method: 'PUT', path: /^\/api\/v1\/prompts\/[^/]+\/labels\/[^/]+$/ }, +]; -function isAllowedWrite(method: string, path: string): boolean { - return WRITE_ROUTES.some((route) => route.method === method && route.path.test(path)); +/** + * A raw path segment the daemon would decode into something other than one + * plain name: `.`/`..` (literal or percent-encoded), an encoded `/` or `\`, + * or an undecodable escape. The allowlist regexes only see raw segments. + */ +function isUnsafeSegment(raw: string): boolean { + if (/%2f|%5c/i.test(raw)) return true; + try { + const decoded = decodeURIComponent(raw); + return decoded === '.' || decoded === '..'; + } catch { + return true; + } +} + +const isAllowedWrite = (req: Request): boolean => + !req.path.split('/').some(isUnsafeSegment) && + OBSERVE_WRITE_ALLOWLIST.some((entry) => entry.method === req.method && entry.path.test(req.path)); + +/** The JSON body to forward (express.json() already parsed it), or `undefined` when it is over the cap. */ +function writeBody(req: Request): string | undefined { + const body = JSON.stringify(req.body ?? {}); + return Buffer.byteLength(body, 'utf8') > MAX_WRITE_BODY_BYTES ? undefined : body; } /** The daemon the Studio reads (and playground runs report to): `LOUSHO_OBSERVE_URL`, default loopback :7777. */ @@ -35,9 +67,14 @@ export function observeTarget(): string { export function registerObserveProxy(app: Express): void { app.use(PROXY_PREFIX, async (req, res) => { - const write = req.method !== 'GET'; - if (write && !isAllowedWrite(req.method, req.path)) { - res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted writes.' }); + const isWrite = req.method !== 'GET'; + if (isWrite && !isAllowedWrite(req)) { + res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted Studio writes.' }); + return; + } + const body = isWrite ? writeBody(req) : undefined; + if (isWrite && body === undefined) { + res.status(413).json({ error: 'Observe write bodies are capped at 1 MiB.' }); return; } // Which daemon this proxy points at, so the down state can name the right LOUSHO_OBSERVE value. @@ -50,8 +87,8 @@ export function registerObserveProxy(app: Express): void { try { response = await fetch(upstream, { method: req.method, - headers: write ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' }, - body: write ? JSON.stringify(req.body ?? {}) : undefined, + headers: isWrite ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' }, + body, signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), }); } catch { diff --git a/apps/agent-forge/src/__tests__/contrast.test.ts b/apps/agent-forge/src/__tests__/contrast.test.ts index 3cb2d452b..61e4aaf4a 100644 --- a/apps/agent-forge/src/__tests__/contrast.test.ts +++ b/apps/agent-forge/src/__tests__/contrast.test.ts @@ -49,6 +49,11 @@ const PAIRS: [text: string, background: string][] = [ ['text', 'surface-2'], // Monitoring: the selected window button draws text on the accent tint. ['text', 'accent-soft'], + // Prompt label chips, the selected DataTable row and the Traces prompt chip (its x in text-muted) on the accent tint. + ['text-muted', 'accent-soft'], + // Prompt version links (runs, Playground) draw --accent on the version card's --surface. The diff's removed/added + // lines reuse danger/danger-soft and success/success-soft, and its line numbers text-faint/surface (all listed above). + ['accent', 'surface'], ]; describe('theme contrast', () => { diff --git a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts index 54ed8cf0b..34b0bd6dd 100644 --- a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts +++ b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts @@ -124,4 +124,51 @@ describe('ObserveClient', () => { expect.anything() ); }); + + it('passes the prompt trace filter through', async () => { + const fetchFn = vi.fn(async () => jsonResponse(200, { traces: [], total: 0, limit: 50, offset: 0 })); + await clientWith(fetchFn as typeof fetch).listTraces({ prompt: 'mira@2' }); + expect(fetchFn).toHaveBeenCalledWith('/observe/api/v1/traces?prompt=mira%402', expect.anything()); + }); + + it('lists prompts and reads one prompt history, undefined on 404', async () => { + const fetchFn = vi.fn(async (url: string) => + url.endsWith('/prompts') + ? jsonResponse(200, { prompts: [{ name: 'mira', version: 2, labels: ['prod'], versions: 2, uses: 3 }] }) + : jsonResponse(404, { error: "No prompt 'x'.", code: 'LOUSHO_REQUEST_INVALID' }) + ); + const client = clientWith(fetchFn as unknown as typeof fetch); + expect((await client.listPrompts())[0]).toMatchObject({ name: 'mira', uses: 3 }); + expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/prompts', expect.anything()); + expect(await client.promptVersions('x y')).toBeUndefined(); + expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/prompts/x%20y', expect.anything()); + }); + + it('movePromptLabel PUTs {version} as JSON with the studio token and returns the move', async () => { + window.sessionStorage.setItem('lousho-studio-token', 'tok-123'); + try { + const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => { + void init; + return jsonResponse(200, { name: 'mira', label: 'prod', version: 2, previous_version: 1 }); + }); + const moved = await clientWith(fetchFn as typeof fetch).movePromptLabel('mira', 'prod', 2); + expect(moved).toEqual({ name: 'mira', label: 'prod', version: 2, previous_version: 1 }); + const [url, init] = fetchFn.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('/observe/api/v1/prompts/mira/labels/prod'); + expect(init.method).toBe('PUT'); + expect(JSON.parse(String(init.body))).toEqual({ version: 2 }); + const headers = new Headers(init.headers); + expect(headers.get('content-type')).toBe('application/json'); + expect(headers.get('x-lousho-studio-token')).toBe('tok-123'); + } finally { + window.sessionStorage.removeItem('lousho-studio-token'); + } + }); + + it('maps write failures like reads: 502 is daemon-down, other errors carry the daemon message', async () => { + const down = clientWith((async () => jsonResponse(502, { error: 'unreachable' })) as typeof fetch); + await expect(down.movePromptLabel('mira', 'prod', 2)).rejects.toBeInstanceOf(ObserveDaemonDownError); + const missing = clientWith((async () => jsonResponse(404, { error: "No prompt 'mira@9'." })) as typeof fetch); + await expect(missing.movePromptLabel('mira', 'prod', 9)).rejects.toMatchObject({ status: 404, message: "No prompt 'mira@9'." }); + }); }); diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts index b35c30ddf..238f5e9d7 100644 --- a/apps/agent-forge/src/observe/observeClient.ts +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -14,6 +14,9 @@ import type { ObserveAgentRow, ObserveHealth, ObserveInsights, + ObservePromptHistory, + ObservePromptLabelMove, + ObservePromptRow, ObserveScorePassRate, ObserveScoreRow, ObserveSessionDetail, @@ -71,6 +74,8 @@ interface TraceListQuery { model?: string; sort?: TraceSort; order?: 'asc' | 'desc'; + /** `` or `@`: traces where any observation ran that registry prompt. */ + prompt?: string; } export interface TraceListFilter extends TraceListQuery { @@ -153,7 +158,7 @@ export class ObserveClient { private async request( path: string, - opts: { query?: Record; method?: 'GET' | 'POST'; body?: unknown } = {} + opts: { query?: Record; method?: 'GET' | 'POST' | 'PUT'; body?: unknown } = {} ): Promise { const headers = new Headers({ accept: 'application/json' }); // The studio token only travels same-origin (to the studio server's @@ -245,6 +250,30 @@ export class ObserveClient { return this.request('/api/v1/insights', { query: { ...filter } }); } + /** `GET /api/v1/prompts` - one row per prompt name (its latest version), with version and use counts. */ + async listPrompts(): Promise { + const body = await this.request<{ prompts: ObservePromptRow[] }>('/api/v1/prompts'); + return body.prompts; + } + + /** `GET /api/v1/prompts/:name` - the version history, newest first; `undefined` on the daemon's 404. */ + async promptVersions(name: string): Promise { + try { + return await this.request(`/api/v1/prompts/${encodeURIComponent(name)}`); + } catch (error) { + if (error instanceof ObserveApiError && error.status === 404) return undefined; + throw error; + } + } + + /** `PUT /api/v1/prompts/:name/labels/:label {version}` - point `label` at `version`, moving it off its old one. */ + movePromptLabel(name: string, label: string, version: number): Promise { + return this.request(`/api/v1/prompts/${encodeURIComponent(name)}/labels/${encodeURIComponent(label)}`, { + method: 'PUT', + body: { version }, + }); + } + /** `GET /api/v1/scores` - eval/judge/human score rows. */ async listScores(filter: ScoreListFilter = {}): Promise { const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { query: { ...filter } }); diff --git a/apps/agent-forge/src/observe/types.ts b/apps/agent-forge/src/observe/types.ts index 95157efb5..f9e6b29a1 100644 --- a/apps/agent-forge/src/observe/types.ts +++ b/apps/agent-forge/src/observe/types.ts @@ -191,3 +191,36 @@ export interface ObserveInsights { agents: ObserveAgentInsights[]; series: ObserveRollupRow[]; } + +/** One version of a registry prompt (`GET /api/v1/prompts/:name` rows; `labels` = the labels pointing at it, sorted). */ +export interface ObservePromptVersion { + prompt_id: string; + name: string; + version: number; + labels: string[]; + variables: unknown; + hash: string | null; + text: string | null; + created_at: number; + /** Observations linked (`prompt_use`) to this version. */ + uses: number; +} + +/** One row of `GET /api/v1/prompts`: the latest version of a name, with its version count and run uses across versions. */ +export interface ObservePromptRow extends ObservePromptVersion { + versions: number; +} + +/** `GET /api/v1/prompts/:name`: the version history, newest first, text included. */ +export interface ObservePromptHistory { + name: string; + versions: ObservePromptVersion[]; +} + +/** `PUT /api/v1/prompts/:name/labels/:label` answer: where the label points now and where it pointed before (null = new label). */ +export interface ObservePromptLabelMove { + name: string; + label: string; + version: number; + previous_version: number | null; +} diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts index cee067f21..3eed3fd24 100644 --- a/apps/agent-forge/src/observe/useObserve.ts +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -11,6 +11,8 @@ import type { ObserveAgentRow, ObserveHealth, ObserveInsights, + ObservePromptHistory, + ObservePromptRow, ObserveScorePassRate, ObserveScoreRow, ObserveSessionDetail, @@ -162,3 +164,11 @@ export function useDaemonDown( }, [down, pollMs]); return down; } + +export function usePrompts(refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listPrompts(), [refreshKey]); +} + +export function usePromptVersions(name: string, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.promptVersions(name), [name, refreshKey]); +} diff --git a/apps/agent-forge/src/screens/prompts/LabelChips.tsx b/apps/agent-forge/src/screens/prompts/LabelChips.tsx new file mode 100644 index 000000000..e6c13eb65 --- /dev/null +++ b/apps/agent-forge/src/screens/prompts/LabelChips.tsx @@ -0,0 +1,13 @@ +/** A prompt version's labels as chips; nothing when it has none. */ +export function LabelChips({ labels }: { labels: readonly string[] }) { + if (labels.length === 0) return null; + return ( +
    + {labels.map((label) => ( +
  • + {label} +
    • + ))} +
+ ); +} diff --git a/apps/agent-forge/src/screens/prompts/PromptDetail.tsx b/apps/agent-forge/src/screens/prompts/PromptDetail.tsx new file mode 100644 index 000000000..cdb6db390 --- /dev/null +++ b/apps/agent-forge/src/screens/prompts/PromptDetail.tsx @@ -0,0 +1,215 @@ +/** + * `#/prompts/:name[@ref]` - one prompt: the selected version (its text, + * labels, "Move