feat(studio): datasets

apps/agent-forge/e2e/a11y.spec.ts

@@ -128,7 +128,7 @@ async function expectNoSeriousAxeViolations(page: Page, label: string) {
 // Every route in the table (new screens are covered automatically): one <main>, one h1,
 // the banner and the primary nav, and an axe scan of the populated (stubbed daemon) state.
 // Plus the deep-link states that render a different view on the same route.
-// Placeholder routes (/evals/:section, /evals/:section/:id) scan the Placeholder today; when their real
+// The datasets routes scan the stub's datasets fixtures. Placeholder routes (/evals/:section, /evals/:section/:id) scan the Placeholder today; when their real
 // screens land, add their endpoints to e2e/observeStub.ts or they get scanned in a 404/empty state.
 const SCANS = [
   ...ROUTES.map(({ pattern, example }) => ({ label: pattern, path: example })),
@@ -171,6 +171,17 @@ test('a #/traces/:id deep link with no daemon shows the down state and keeps the
   expect(new URL(page.url()).hash).toBe('#/traces/e2e-trace?obs=e2e-obs');
 });
 
+test('#/evals/datasets/:name with no daemon shows the down state, keeps the URL and the Evaluations tabs', async ({ page }) => {
+  await page.route('**/observe/**', (route) =>
+    route.fulfill({ status: 502, contentType: 'application/json', body: '{"error":"down"}' }),
+  );
+  await page.goto('/?token=e2e-studio-token#/evals/datasets/golden?offset=50');
+  await expect(page.getByRole('main').getByText('lousho observe start')).toBeVisible();
+  const tabs = page.getByRole('navigation', { name: 'Evaluations' });
+  await expect(tabs.locator('[aria-current="page"]')).toHaveText('Datasets');
+  expect(new URL(page.url()).hash).toBe('#/evals/datasets/golden?offset=50');
+});
+
 test('#/monitoring with no daemon shows the down state and keeps window/agent in the URL', async ({ page }) => {
   await page.route('**/observe/**', (route) =>
     route.fulfill({ status: 502, contentType: 'application/json', body: '{"error":"down"}' }),

apps/agent-forge/e2e/observeStub.ts

@@ -1,6 +1,8 @@
 import type { Page, Route } from '@playwright/test';
 import type {
   ObserveAgentRow,
+  ObserveDataset,
+  ObserveExample,
   ObserveHubEntry,
   ObserveInsights,
   ObservePromptVersion,
@@ -14,8 +16,8 @@ import type {
  * A fake observe daemon behind the studio's `/observe` proxy, so the e2e specs
  * scan the screens' populated states rather than "daemon down". One agent, one
  * thread with one trace (an LLM call and a tool call), one prompt with two
- * versions, an eval's scores, and a Context Hub namespace with a fleet and a
- * private entry.
+ * versions, an eval's scores, a Context Hub namespace with a fleet and a
+ * private entry, and a dataset with two examples (one copied from the trace).
  */
 const NOW = Date.now();
 const T0 = NOW - 60_000;
@@ -174,6 +176,32 @@ const hubEntry = (key: string, value: string, scope: ObserveHubEntry['scope'], t
 });
 const hubEntries = [hubEntry('order-policy', '{"returns_days":30}', 'fleet', null), hubEntry('last-customer', 'cust-42', 'private', 24 * HOUR)];
 
+const dataset: ObserveDataset = {
+  dataset_id: 'e2e-dataset-id',
+  name: 'e2e-dataset',
+  description: 'Support cases the agent must get right',
+  created_at: T0 - HOUR,
+  example_count: 2,
+};
+const examples: ObserveExample[] = [
+  {
+    example_id: 'e2e-example-1',
+    dataset_id: dataset.dataset_id,
+    input: 'Where is my order?',
+    expected: 'It ships tomorrow.',
+    source_trace_id: 'e2e-trace',
+    created_at: T0,
+  },
+  {
+    example_id: 'e2e-example-2',
+    dataset_id: dataset.dataset_id,
+    input: [{ role: 'user', content: 'Can I return it?' }],
+    expected: null,
+    source_trace_id: null,
+    created_at: T0 + 1_000,
+  },
+];
+
 const json = (route: Route, body: unknown, status = 200) =>
   route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) });
 
@@ -201,6 +229,10 @@ export async function stubObserve(page: Page): Promise<void> {
       const scope = q.get('scope');
       return json(route, { entries: hubEntries.filter((e) => (!ns || e.namespace === ns) && (!scope || e.scope === scope)) });
     }
+    if (path === '/api/v1/datasets' && route.request().method() === 'GET')
+      return json(route, { datasets: [dataset], total: 1, limit: Number(q.get('limit') ?? 100), offset: 0 });
+    if (path === '/api/v1/datasets/e2e-dataset')
+      return json(route, { dataset, examples, total: examples.length, limit: Number(q.get('limit') ?? 1000), offset: 0 });
     if (path === '/api/v1/scores') {
       if (q.get('aggregate') === 'eval')
         return json(route, { evals: [{ name: 'e2e-eval', count: 4, passed: 3, pass_rate: 0.75, avg_value: 0.75, last_ts: T0 + 2_000 }] });

apps/agent-forge/server/__tests__/observeProxy.test.ts

@@ -3,7 +3,8 @@
  * panels' GETs same-origin, because the daemon itself serves no CORS
  * headers. A dead daemon answers 502 with the `lousho observe start` hint.
  * Writes go through only when OBSERVE_WRITE_ALLOWLIST names the method and
- * path (feedback scores, a prompt label move, a hub publish); every other non-GET stays 405.
+ * path (feedback scores, a prompt label move, a hub publish, dataset and
+ * example writes); every other non-GET stays 405.
  */
 import { describe, it, expect, beforeEach, afterEach } from 'vitest';
 import * as http from 'node:http';
@@ -24,6 +25,7 @@ describe('observe proxy', () => {
   let app: ReturnType<typeof createApp>;
   let daemon: http.Server | undefined;
   let daemonUrl: string;
+  let lastDelete: { url: string | undefined; contentType: string | null; body: string } | undefined;
   const prevEnv = process.env.LOUSHO_OBSERVE_URL;
 
   beforeEach(async () => {
@@ -43,6 +45,12 @@ describe('observe proxy', () => {
       let body = '';
       req.on('data', (chunk: Buffer) => (body += chunk.toString()));
       req.on('end', () => {
+        if (req.method === 'DELETE') {
+          lastDelete = { url: req.url, contentType: req.headers['content-type'] ?? null, body };
+          res.writeHead(204);
+          res.end();
+          return;
+        }
         res.writeHead(req.method === 'POST' ? 201 : 200, { 'Content-Type': 'application/json' });
         res.end(
           JSON.stringify({
@@ -117,6 +125,49 @@ describe('observe proxy', () => {
     expect(res.body.body).toEqual({ version: 2 });
   });
 
+  it('forwards POST /api/v1/datasets (a new dataset) with its JSON body', async () => {
+    const res = await request(app).post('/observe/api/v1/datasets').send({ name: 'golden', description: 'd' });
+    expect(res.status).toBe(201);
+    expect(res.body).toEqual({
+      echoed: '/api/v1/datasets',
+      method: 'POST',
+      contentType: 'application/json',
+      body: { name: 'golden', description: 'd' },
+    });
+  });
+
+  it('forwards POST /api/v1/datasets/:name/examples (add from a trace) with its JSON body', async () => {
+    const res = await request(app).post('/observe/api/v1/datasets/golden/examples').send({ from_trace_id: 't1' });
+    expect(res.status).toBe(201);
+    expect(res.body.echoed).toBe('/api/v1/datasets/golden/examples');
+    expect(res.body.body).toEqual({ from_trace_id: 't1' });
+  });
+
+  it('forwards DELETE /api/v1/datasets/:name/examples/:id with no body and passes the 204 back', async () => {
+    lastDelete = undefined;
+    const res = await request(app).delete('/observe/api/v1/datasets/golden/examples/e1');
+    expect(res.status).toBe(204);
+    expect(res.text).toBe('');
+    expect(lastDelete).toEqual({ url: '/api/v1/datasets/golden/examples/e1', contentType: null, body: '' });
+  });
+
+  it.each([
+    ['delete', '/observe/api/v1/datasets/golden'],
+    ['delete', '/observe/api/v1/datasets/golden/examples'],
+    ['put', '/observe/api/v1/datasets/golden/examples/e1'],
+    ['post', '/observe/api/v1/datasets/golden/examples/e1'],
+    ['post', '/observe/api/v1/datasets/golden'],
+    ['delete', '/observe/api/v1/traces/t1'],
+  ] as const)('keeps the dataset-adjacent write %s %s at 405', async (method, url) => {
+    const res = await request(app)[method](url).send({});
+    expect(res.status).toBe(405);
+  });
+
+  it('keeps a DELETE whose example segment is a dot or an encoded slash at 405', async () => {
+    expect((await request(app).delete('/observe/api/v1/datasets/golden/examples/%2e%2e')).status).toBe(405);
+    expect((await request(app).delete('/observe/api/v1/datasets/a%2Fb/examples/e1')).status).toBe(405);
+  });
+
   it.each([
     ['post', '/observe/api/v1/prompts'],
     ['put', '/observe/api/v1/prompts/mira'],
@@ -150,8 +201,8 @@ describe('observe proxy', () => {
     expect(res.status).toBe(413);
   });
 
-  it('exports one write allowlist of method + path pairs: feedback scores, the label move and hub publishes', () => {
-    expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT', 'POST']);
+  it('exports one write allowlist of method + path pairs: feedback scores, the label move, hub publishes and dataset writes', () => {
+    expect(OBSERVE_WRITE_ALLOWLIST.map((entry) => entry.method)).toEqual(['POST', 'PUT', 'POST', 'POST', 'POST', 'DELETE']);
     expect(OBSERVE_WRITE_ALLOWLIST.every((entry) => entry.path instanceof RegExp)).toBe(true);
   });
 });

apps/agent-forge/server/observeProxy.ts

@@ -11,8 +11,9 @@
  * GET is forwarded for every path. Writes are forwarded only when
  * {@link OBSERVE_WRITE_ALLOWLIST} names the method and path - a short list of
  * Studio actions (human feedback scores, moving a prompt label, editing a
- * Context Hub fleet entry) - with the JSON body app.ts's `express.json`
- * already parsed, re-serialised and capped at 1 MiB. Every other non-GET - the ingest endpoint above all - answers 405
+ * Context Hub fleet entry, dataset and example writes) - with the JSON body
+ * app.ts's `express.json` already parsed, re-serialised and capped at 1 MiB
+ * (a DELETE forwards no body, and its 204 passes back empty). Every other non-GET - the ingest endpoint above all - answers 405
  * and stays reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it
  * carries the same per-launch token requirement as the rest of the studio API.
  */
@@ -25,7 +26,7 @@ const MAX_WRITE_BODY_BYTES = 1024 * 1024;
 
 /** A daemon write the proxy forwards: the method plus a regex over the path after `/observe` (no query). */
 export interface ObserveWrite {
-  method: 'POST' | 'PUT';
+  method: 'POST' | 'PUT' | 'DELETE';
   path: RegExp;
 }
 
@@ -34,6 +35,9 @@ export const OBSERVE_WRITE_ALLOWLIST: readonly ObserveWrite[] = [
   { method: 'POST', path: /^\/api\/v1\/scores$/ },
   { method: 'PUT', path: /^\/api\/v1\/prompts\/[^/]+\/labels\/[^/]+$/ },
   { method: 'POST', path: /^\/api\/v1\/hub$/ },
+  { method: 'POST', path: /^\/api\/v1\/datasets$/ },
+  { method: 'POST', path: /^\/api\/v1\/datasets\/[^/]+\/examples$/ },
+  { method: 'DELETE', path: /^\/api\/v1\/datasets\/[^/]+\/examples\/[^/]+$/ },
 ];
 
 /**
@@ -73,8 +77,9 @@ export function registerObserveProxy(app: Express): void {
       res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted Studio writes.' });
       return;
     }
-    const body = isWrite ? writeBody(req) : undefined;
-    if (isWrite && body === undefined) {
+    const hasBody = isWrite && req.method !== 'DELETE';
+    const body = hasBody ? writeBody(req) : undefined;
+    if (hasBody && body === undefined) {
       res.status(413).json({ error: 'Observe write bodies are capped at 1 MiB.' });
       return;
     }
@@ -88,7 +93,7 @@ export function registerObserveProxy(app: Express): void {
     try {
       response = await fetch(upstream, {
         method: req.method,
-        headers: isWrite ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' },
+        headers: hasBody ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' },
         body,
         signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
       });
@@ -99,6 +104,10 @@ export function registerObserveProxy(app: Express): void {
       return;
     }
     res.status(response.status);
+    if (response.status === 204) {
+      res.end();
+      return;
+    }
     const contentType = response.headers.get('content-type');
     if (contentType) res.setHeader('content-type', contentType);
     res.send(Buffer.from(await response.arrayBuffer()));

apps/agent-forge/src/observe/__tests__/observeClient.test.ts

@@ -193,4 +193,36 @@ describe('ObserveClient', () => {
     expect(init.method).toBe('POST');
     expect(JSON.parse(String(init.body))).toEqual(body);
   });
+  it('reads datasets: the paged list, one dataset with its examples page, undefined on 404', async () => {
+    const fetchFn = vi.fn(async (url: string) =>
+      url.includes('/datasets/nope')
+        ? jsonResponse(404, { error: "No dataset 'nope'.", code: 'LOUSHO_REQUEST_INVALID' })
+        : jsonResponse(200, { datasets: [], total: 0, limit: 50, offset: 0 })
+    );
+    const client = clientWith(fetchFn as unknown as typeof fetch);
+    await client.listDatasets({ limit: 50, offset: 100 });
+    expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/datasets?limit=50&offset=100', expect.anything());
+    await client.dataset('golden.v1', { limit: 50, offset: 0 });
+    expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/datasets/golden.v1?limit=50&offset=0', expect.anything());
+    expect(await client.dataset('nope')).toBeUndefined();
+  });
+
+  it('writes datasets: create and add-example POST JSON, delete-example sends a bodiless DELETE and takes the 204', async () => {
+    const fetchFn = vi.fn(async (_url: string, init?: RequestInit) =>
+      init?.method === 'DELETE' ? new Response(null, { status: 204 }) : jsonResponse(201, { dataset: {}, example: {} })
+    );
+    const client = clientWith(fetchFn as unknown as typeof fetch);
+    await client.createDataset({ name: 'golden' });
+    expect(fetchFn).toHaveBeenLastCalledWith('/observe/api/v1/datasets', expect.objectContaining({ method: 'POST', body: '{"name":"golden"}' }));
+    await client.addExample('golden', { from_trace_id: 't1' });
+    expect(fetchFn).toHaveBeenLastCalledWith(
+      '/observe/api/v1/datasets/golden/examples',
+      expect.objectContaining({ method: 'POST', body: '{"from_trace_id":"t1"}' })
+    );
+    await expect(client.deleteExample('golden', 'e/1')).resolves.toBeUndefined();
+    const [url, init] = fetchFn.mock.calls.at(-1)!;
+    expect(url).toBe('/observe/api/v1/datasets/golden/examples/e%2F1');
+    expect(init?.method).toBe('DELETE');
+    expect(init?.body).toBeUndefined();
+  });
 });

apps/agent-forge/src/observe/observeClient.ts

@@ -13,6 +13,10 @@ import { STUDIO_TOKEN_HEADER, loadStudioToken } from '../runtime/studioToken';
 import type {
   ObserveAgentRow,
   NewHubEntry,
+  ObserveDataset,
+  ObserveDatasetDetail,
+  ObserveDatasetList,
+  ObserveExample,
   ObserveHealth,
   ObserveHubEntry,
   ObserveHubScope,
@@ -138,6 +142,15 @@ export interface HubEntryFilter {
   limit?: number;
 }
 
+/** A page request (`limit`/`offset`) for the dataset list and a dataset's examples. */
+export interface PageQuery {
+  limit?: number;
+  offset?: number;
+}
+
+/** The body of {@link ObserveClient.addExample}: a literal example, or one copied from a trace's root input/output. */
+export type NewExample = { input: unknown; expected?: unknown } | { from_trace_id: string };
+
 export interface InsightsFilter {
   days?: number;
   since?: number;
@@ -169,7 +182,7 @@ export class ObserveClient {
 
   private async request<T>(
     path: string,
-    opts: { query?: Record<string, string | number | undefined>; method?: 'GET' | 'POST' | 'PUT'; body?: unknown } = {}
+    opts: { query?: Record<string, string | number | undefined>; method?: 'GET' | 'POST' | 'PUT' | 'DELETE'; body?: unknown } = {}
   ): Promise<T> {
     const headers = new Headers({ accept: 'application/json' });
     // The studio token only travels same-origin (to the studio server's
@@ -195,6 +208,8 @@ export class ObserveClient {
       const body = (await res.json().catch(() => undefined)) as { error?: string } | undefined;
       throw new ObserveApiError(body?.error || `Request to ${path} failed with ${res.status}`, res.status);
     }
+    // 204 No Content (an example delete) has no body to parse.
+    if (res.status === 204) return undefined as T;
     return (await res.json()) as T;
   }
 
@@ -315,6 +330,43 @@ export class ObserveClient {
     return this.request('/api/v1/scores', { method: 'POST', body: score });
   }
 
+  /** `GET /api/v1/datasets` - a page of datasets (with example counts), newest first. */
+  listDatasets(page: PageQuery = {}): Promise<ObserveDatasetList> {
+    return this.request('/api/v1/datasets', { query: { ...page } });
+  }
+
+  /** `GET /api/v1/datasets/:name` - the dataset plus a page of its examples, oldest first; `undefined` on the daemon's 404. */
+  async dataset(name: string, page: PageQuery = {}): Promise<ObserveDatasetDetail | undefined> {
+    try {
+      return await this.request(`/api/v1/datasets/${encodeURIComponent(name)}`, { query: { ...page } });
+    } catch (error) {
+      if (error instanceof ObserveApiError && error.status === 404) return undefined;
+      throw error;
+    }
+  }
+
+  /** `POST /api/v1/datasets` - a new, empty dataset; a taken name is the daemon's 409 (`ObserveApiError`). */
+  async createDataset(input: { name: string; description?: string }): Promise<ObserveDataset> {
+    const body = await this.request<{ dataset: ObserveDataset }>('/api/v1/datasets', { method: 'POST', body: input });
+    return body.dataset;
+  }
+
+  /** `POST /api/v1/datasets/:name/examples` - one example, literal or `{ from_trace_id }` (the trace root's input/output). */
+  async addExample(name: string, example: NewExample): Promise<ObserveExample> {
+    const body = await this.request<{ example: ObserveExample }>(`/api/v1/datasets/${encodeURIComponent(name)}/examples`, {
+      method: 'POST',
+      body: example,
+    });
+    return body.example;
+  }
+
+  /** `DELETE /api/v1/datasets/:name/examples/:id` - removes one example (204). */
+  async deleteExample(name: string, exampleId: string): Promise<void> {
+    await this.request(`/api/v1/datasets/${encodeURIComponent(name)}/examples/${encodeURIComponent(exampleId)}`, {
+      method: 'DELETE',
+    });
+  }
+
   /** `GET /api/v1/prompts/:name/:ref` (ref = `latest`, a version number or a label); `undefined` on the daemon's 404. */
   async getPrompt(name: string, ref = 'latest'): Promise<ObservePrompt | undefined> {
     try {