Skip to content

Commit b7bbb70

Browse files
LinuxDevilclaude
andauthored
docs(studio): document the new navigation and screens (#765)
* docs(studio): document the new navigation and screens agent-forge.md: the left nav, a deep-link route table, keyboard shortcuts, status words, the collector item and daemon-down behaviour, and a section per screen (Agents, Playground, Traces, Trace, Threads, Monitoring, Scores, Datasets, Evaluators, Annotation queues, Prompts, Context Hub, Settings), with screenshots. observability.md, cli.md and evals.md link to the screens. The Evaluators section is written from the API and marked for F21. Screenshots come from the opt-in e2e/docShots.spec.ts (LOUSHO_DOC_SHOTS=1) against the seeded fake daemon. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(studio): correct how the Studio proxy finds and authenticates to the daemon Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(upgrading): observe round 2 behaviour changes Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(studio): document the Evaluators screen Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent f314110 commit b7bbb70

20 files changed

Lines changed: 1078 additions & 135 deletions
Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
import { test, expect, type Page, type Route } from '@playwright/test';
2+
import * as fs from 'node:fs';
3+
import * as path from 'node:path';
4+
import { fileURLToPath } from 'node:url';
5+
import type { ObserveAgentRow, ObserveInsights, ObserveSessionRow } from '../src/observe/types';
6+
import { stubObserve } from './observeStub';
7+
import { SEED_ERROR, SEED_NEWEST_FAILURE, seedObserve } from './fixtures/seedObserve';
8+
9+
/**
10+
* The screenshots in docs/agent-forge.md, taken from the built Studio against
11+
* the seeded fake daemon (`observeStub.ts` + `fixtures/seedObserve.ts`). Opt-in:
12+
*
13+
* LOUSHO_DOC_SHOTS=1 npx playwright test -c apps/agent-forge/playwright.config.ts docShots
14+
*
15+
* writes `docs/images/studio-<screen>.png` at the repo root (1280x800, light).
16+
* The small fixtures below fill the screens the seed leaves thin: a fleet with
17+
* mixed health, a day of chart buckets for it, and a few threads.
18+
*/
19+
test.skip(!process.env.LOUSHO_DOC_SHOTS, 'set LOUSHO_DOC_SHOTS=1 to regenerate the docs screenshots');
20+
test.use({ viewport: { width: 1280, height: 800 }, colorScheme: 'light' });
21+
22+
const APP = '/?token=e2e-studio-token';
23+
const OUT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../docs/images');
24+
25+
const NOW = Date.now();
26+
const MINUTE = 60_000;
27+
const HOUR = 60 * MINUTE;
28+
29+
const FLEET = [
30+
{ id: 'orders-bot', name: 'Orders bot', project: 'shop', status: 'healthy', age: 4_000, runs: 21, errorRate: 0.19, p95: 1_320 },
31+
{ id: 'billing-bot', name: 'Billing bot', project: 'shop', status: 'healthy', age: 9_000, runs: 20, errorRate: 0.2, p95: 980 },
32+
{ id: 'triage-bot', name: 'Triage bot', project: 'support', status: 'stale', age: 41_000, runs: 19, errorRate: 0.21, p95: 2_600 },
33+
] as const;
34+
35+
const agents: ObserveAgentRow[] = FLEET.map((a) => ({
36+
agent_id: a.id,
37+
project: a.project,
38+
name: a.name,
39+
version: '1.0.0',
40+
first_seen: NOW - 48 * HOUR,
41+
last_heartbeat: NOW - a.age,
42+
status: a.status,
43+
heartbeat_age_ms: a.age,
44+
heartbeat_interval_ms: 15_000,
45+
hostname: 'seed-host',
46+
pid: 4242,
47+
dropped: 0,
48+
runs_today: a.runs,
49+
error_rate_today: a.errorRate,
50+
}));
51+
52+
/** 18 hourly buckets per agent; a wave so the charts have shape. */
53+
function series(): ObserveInsights['series'] {
54+
const rows: ObserveInsights['series'] = [];
55+
FLEET.forEach((a, n) => {
56+
for (let h = 17; h >= 0; h -= 1) {
57+
const bucket = Math.floor((NOW - h * HOUR) / HOUR) * HOUR;
58+
const runs = 2 + ((h + n * 3) % 4);
59+
const errors = h % 5 === n ? 1 : 0;
60+
const values: Record<string, number> = {
61+
runs,
62+
errors,
63+
cost_usd: runs * 0.0009,
64+
input_tokens: runs * 720,
65+
output_tokens: runs * 55,
66+
p95_latency_ms: a.p95 * (0.7 + ((h + n) % 4) * 0.12),
67+
error_rate: errors / runs,
68+
};
69+
for (const [metric, value] of Object.entries(values))
70+
rows.push({ bucket_start: bucket, granularity: 'hour', agent_id: a.id, metric, value });
71+
}
72+
});
73+
return rows;
74+
}
75+
76+
const insights = (): ObserveInsights => ({
77+
window: { since: NOW - 24 * HOUR, until: NOW },
78+
agents: FLEET.map((a) => ({
79+
agent_id: a.id,
80+
project: a.project,
81+
name: a.name,
82+
runs: a.runs,
83+
runs_per_day: a.runs,
84+
errors: Math.round(a.runs * a.errorRate),
85+
error_rate: a.errorRate,
86+
cost_usd: a.runs * 0.0009,
87+
input_tokens: a.runs * 720,
88+
output_tokens: a.runs * 55,
89+
p50_latency_ms: a.p95 / 2,
90+
p95_latency_ms: a.p95,
91+
})),
92+
series: series(),
93+
});
94+
95+
const sessions: ObserveSessionRow[] = [
96+
['seed-session-0', 0, 'active', 'Where is order ord_54?', 9],
97+
['seed-session-1', 1, 'active', 'Why was I charged twice?', 6],
98+
['seed-session-2', 2, 'ended', 'My parcel arrived damaged', 4],
99+
['seed-session-3', 0, 'ended', 'Can I change the delivery address?', 3],
100+
['seed-session-4', 1, 'interrupted', 'Refund the last invoice', 2],
101+
].map(([id, agent, status, input, traces], i) => {
102+
const a = FLEET[agent as number];
103+
const started = NOW - (i + 1) * 2 * HOUR;
104+
return {
105+
session_id: id as string,
106+
agent_id: a.id,
107+
agent_name: a.name,
108+
status: status as string,
109+
started_at: started,
110+
ended_at: started + (traces as number) * 4 * MINUTE,
111+
traces: traces as number,
112+
input_tokens: (traces as number) * 720,
113+
output_tokens: (traces as number) * 55,
114+
cost_usd: (traces as number) * 0.0009,
115+
last_input_preview: input as string,
116+
};
117+
});
118+
119+
const json = (route: Route, body: unknown) => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) });
120+
121+
/** The doc-only fixtures; registered after seedObserve, so they answer first. */
122+
async function docFixtures(page: Page): Promise<void> {
123+
await page.route('**/observe/api/v1/**', (route) => {
124+
if (route.request().method() !== 'GET') return route.fallback();
125+
const p = new URL(route.request().url()).pathname.replace(/^\/observe/, '');
126+
if (p === '/api/v1/agents') return json(route, { agents });
127+
if (p === '/api/v1/insights') return json(route, insights());
128+
if (p === '/api/v1/sessions') return json(route, { sessions, total: sessions.length, limit: 50, offset: 0 });
129+
return route.fallback();
130+
});
131+
}
132+
133+
test.beforeAll(() => {
134+
fs.mkdirSync(OUT, { recursive: true });
135+
});
136+
137+
test.beforeEach(async ({ page }) => {
138+
await stubObserve(page);
139+
await seedObserve(page);
140+
await docFixtures(page);
141+
});
142+
143+
async function open(page: Page, hash: string) {
144+
await page.goto(`${APP}#${hash}`);
145+
await expect(page.locator('.app')).toBeVisible();
146+
}
147+
148+
async function shoot(page: Page, screen: string) {
149+
// Let toasts, focus rings and chart transitions settle.
150+
await page.mouse.move(0, 0);
151+
await page.waitForTimeout(300);
152+
await page.screenshot({ path: path.join(OUT, `studio-${screen}.png`) });
153+
}
154+
155+
const rows = (page: Page) => page.getByRole('main').locator('table.data-table tbody tr');
156+
157+
test('agents: the builder with an agent', async ({ page }) => {
158+
await open(page, '/agents');
159+
await page.getByRole('button', { name: '+ New agent' }).click();
160+
await page.locator('#new-agent-name').fill('support-bot');
161+
await page.locator('#new-agent-template').selectOption('support-bot');
162+
await page.getByRole('button', { name: 'Create', exact: true }).click();
163+
await expect(page.locator('[data-node-type="tool"]')).toBeVisible();
164+
await shoot(page, 'agents');
165+
});
166+
167+
test('traces: the list', async ({ page }) => {
168+
await open(page, '/traces');
169+
await expect(rows(page).first()).toBeVisible();
170+
await shoot(page, 'traces');
171+
});
172+
173+
test('trace: a failed run with its error banner', async ({ page }) => {
174+
await open(page, `/traces/${SEED_NEWEST_FAILURE.trace_id}`);
175+
await expect(page.getByRole('main').getByRole('alert')).toContainText(SEED_ERROR);
176+
await shoot(page, 'trace');
177+
});
178+
179+
test('threads: the session list', async ({ page }) => {
180+
await open(page, '/threads');
181+
await expect(rows(page)).toHaveCount(sessions.length);
182+
await shoot(page, 'threads');
183+
});
184+
185+
test('monitoring: fleet and charts', async ({ page }) => {
186+
await open(page, '/monitoring');
187+
await expect(page.locator('.mon-chart').first()).toBeVisible();
188+
await expect(page.getByRole('region', { name: 'Fleet' }).locator('tbody tr')).toHaveCount(FLEET.length);
189+
await shoot(page, 'monitoring');
190+
});
191+
192+
test('scores: an expanded eval', async ({ page }) => {
193+
await open(page, '/evals/scores?open=e2e-eval');
194+
await expect(page.getByRole('region', { name: 'Eval e2e-eval' })).toBeVisible();
195+
await shoot(page, 'scores');
196+
});
197+
198+
test('datasets: one dataset with its examples', async ({ page }) => {
199+
await open(page, '/evals/datasets/e2e-dataset');
200+
await expect(rows(page)).toHaveCount(2);
201+
await shoot(page, 'datasets');
202+
});
203+
204+
test('evaluators: the rule list', async ({ page }) => {
205+
await open(page, '/evals/evaluators');
206+
await expect(rows(page)).toHaveCount(2);
207+
await expect(page.getByRole('switch', { name: 'answered enabled' })).toBeChecked();
208+
await shoot(page, 'evaluators');
209+
});
210+
211+
test('annotation queue: reviewing an item', async ({ page }) => {
212+
await open(page, '/evals/queues/e2e-queue');
213+
await expect(page.locator('[data-review-trace="e2e-trace"]')).toBeVisible();
214+
await shoot(page, 'queue-review');
215+
});
216+
217+
test('prompts: a prompt with labels and a diff', async ({ page }) => {
218+
await open(page, '/prompts/e2e-prompt?diff=1,2');
219+
await expect(page.getByRole('region', { name: 'Compare versions' })).toBeVisible();
220+
await shoot(page, 'prompts');
221+
});
222+
223+
test('playground: loaded from a trace generation', async ({ page }) => {
224+
await open(page, '/playground?trace=e2e-trace&fromObs=e2e-obs');
225+
const typed = () => page.locator('main textarea').evaluateAll((els) => els.map((e) => (e as HTMLTextAreaElement).value).join('|'));
226+
await expect.poll(typed).toContain('Where is my order?');
227+
await shoot(page, 'playground');
228+
});
229+
230+
test('command palette: open over Traces', async ({ page }) => {
231+
await open(page, '/traces');
232+
await expect(rows(page).first()).toBeVisible();
233+
await page.keyboard.press('Control+K');
234+
const input = page.getByRole('combobox', { name: 'Command' });
235+
await expect(input).toBeFocused();
236+
await input.fill('go');
237+
await page.waitForTimeout(150);
238+
await page.screenshot({ path: path.join(OUT, 'studio-command-palette.png') });
239+
});

0 commit comments

Comments
 (0)