diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index 63bd46d0..b406ad51 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -1,4 +1,7 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, type Page } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; +import { ROUTES } from '../src/screens/routeTable'; +import { stubObserve } from './observeStub'; /** * Keyboard and screen-reader access to the studio - landmarks @@ -6,6 +9,8 @@ import { test, expect } from '@playwright/test'; * palette/hook/toggle controls that work from the keyboard alone. */ test.beforeEach(async ({ page }) => { + // The stub goes in before the first load, so shell-level fetches (health) see a live daemon too. + await stubObserve(page); await page.goto('/?token=e2e-studio-token'); await expect(page.locator('.app')).toBeVisible(); }); @@ -112,20 +117,50 @@ test('Ctrl+K opens the command palette; ? lists the shortcuts', async ({ page }) await expect(sheet).toHaveCount(0); }); -// One
, one h1, the banner and the primary nav on every screen, not just Agents. -for (const route of ['#/traces', '#/traces?status=error', '#/traces/e2e-trace?obs=e2e-obs', '#/threads', '#/threads/s1', '#/threads?status=active', '#/monitoring', '#/evals/scores', '#/prompts', '#/prompts/mira@2?diff=1,2', '#/hub', '#/hub/docs?scope=fleet', '#/settings', '#/playground']) { - test(`${route} keeps the landmarks and exactly one main`, async ({ page }) => { - await page.goto(`/?token=e2e-studio-token${route}`); - await expect(page.locator('.screen-main')).toBeVisible(); +/** Run axe (WCAG 2 A/AA) on the page as it stands; fail on any serious/critical violation, naming the rule and its targets. */ +async function expectNoSeriousAxeViolations(page: Page, label: string) { + const { violations } = await new AxeBuilder({ page }).withTags(['wcag2a', 'wcag2aa']).analyze(); + const serious = violations.filter((v) => v.impact === 'serious' || v.impact === 'critical'); + const report = serious.map((v) => `${v.id} (${v.impact}): ${v.nodes.map((n) => n.target.join(' ')).join(', ')}`); + expect(report, `axe violations on ${label}`).toEqual([]); +} + +// Every route in the table (new screens are covered automatically): one
, one h1, +// the banner and the primary nav, and an axe scan of the populated (stubbed daemon) state. +// Plus the deep-link states that render a different view on the same route. +// Placeholder routes (/evals/:section, /evals/:section/:id) scan the Placeholder today; when their real +// screens land, add their endpoints to e2e/observeStub.ts or they get scanned in a 404/empty state. +const SCANS = [ + ...ROUTES.map(({ pattern, example }) => ({ label: pattern, path: example })), + ...['/traces?status=error', '/traces/e2e-trace?obs=e2e-obs', '/threads?status=active', '/prompts/e2e-prompt?diff=1,2'].map((path) => ({ label: path, path })), +]; +for (const { label, path } of SCANS) { + test(`${label} keeps the landmarks and passes axe`, async ({ page }) => { + await page.goto(`/?token=e2e-studio-token#${path}`); + await expect(page.locator('.app')).toBeVisible(); + await page.waitForLoadState('networkidle'); await expect(page.getByRole('main')).toHaveCount(1); await expect(page.locator('main')).toHaveCount(1); await expect(page.getByRole('banner')).toHaveCount(1); await expect(page.getByRole('navigation', { name: 'Primary' })).toHaveCount(1); await expect(page.getByRole('heading', { level: 1 })).toHaveText('Agent Forge'); await expect(page.getByRole('navigation', { name: 'Primary' }).locator('[aria-current="page"]')).toHaveCount(1); + await expectNoSeriousAxeViolations(page, path); }); } +test('the command palette passes axe', async ({ page }) => { + await page.keyboard.press('Control+k'); + await expect(page.getByRole('dialog', { name: 'Command palette' })).toBeVisible(); + await expectNoSeriousAxeViolations(page, 'command palette'); +}); + +test('the shortcut sheet passes axe', async ({ page }) => { + await page.keyboard.press('?'); + await expect(page.getByRole('dialog', { name: 'Keyboard shortcuts' })).toBeVisible(); + await expectNoSeriousAxeViolations(page, 'shortcut sheet'); +}); + test('a #/traces/:id deep link with no daemon shows the down state and keeps the URL', async ({ page }) => { // Answer like the studio proxy does for a dead daemon, whatever runs on the machine. await page.route('**/observe/**', (route) => @@ -159,4 +194,27 @@ test('#/settings shows the settings panel inside the screen', async ({ page }) = const main = page.getByRole('main'); await expect(main.getByRole('heading', { name: 'Settings' })).toBeAttached(); await expect(main.getByRole('combobox', { name: 'Active profile' })).toBeVisible(); + const otel = main.getByRole('switch', { name: 'Enable OTel tracing' }); + const checked = await otel.getAttribute('aria-checked'); + await otel.focus(); + await page.keyboard.press('Space'); + await expect(otel).toHaveAttribute('aria-checked', checked === 'true' ? 'false' : 'true'); +}); + +test('the axe scans see the stubbed daemon data, not the down state', async ({ page }) => { + const main = page.getByRole('main'); + for (const [path, text] of [ + ['/traces', 'Where is my order?'], + ['/traces/e2e-trace', 'lookup_order'], + ['/threads/e2e-session', 'Where is my order?'], + ['/monitoring', 'Support bot'], + ['/evals/scores', 'e2e-eval'], + ['/prompts/e2e-prompt', 'how can I help today?'], + ['/hub', 'e2e-namespace'], + ['/hub/e2e-namespace', 'order-policy'], + ]) { + await page.goto(`/?token=e2e-studio-token#${path}`); + await expect(main.getByText(text).first(), path).toBeVisible(); + await expect(main.getByText('lousho observe start'), path).toHaveCount(0); + } }); diff --git a/apps/agent-forge/e2e/observeStub.ts b/apps/agent-forge/e2e/observeStub.ts new file mode 100644 index 00000000..1581b6f3 --- /dev/null +++ b/apps/agent-forge/e2e/observeStub.ts @@ -0,0 +1,211 @@ +import type { Page, Route } from '@playwright/test'; +import type { + ObserveAgentRow, + ObserveHubEntry, + ObserveInsights, + ObservePromptVersion, + ObserveScoreRow, + ObserveSessionRow, + ObserveTraceDetail, + ObserveTraceRow, +} from '../src/observe/types'; + +/** + * A fake observe daemon behind the studio's `/observe` proxy, so the e2e specs + * scan the screens' populated states rather than "daemon down". One agent, one + * thread with one trace (an LLM call and a tool call), one prompt with two + * versions, an eval's scores, and a Context Hub namespace with a fleet and a + * private entry. + */ +const NOW = Date.now(); +const T0 = NOW - 60_000; + +const trace: ObserveTraceRow = { + trace_id: 'e2e-trace', + name: 'support turn', + agent_id: 'e2e-agent', + session_id: 'e2e-session', + start: T0, + end: T0 + 1_800, + status: 'ok', + observations: 3, + generations: 1, + tools: 1, + errors: 0, + input_tokens: 120, + output_tokens: 48, + cost_usd: 0.0012, + agent_name: 'Support bot', + model: 'mock-echo', + error_message: null, + input_preview: 'Where is my order?', + output_preview: 'It ships tomorrow.', +}; + +const failedTrace: ObserveTraceRow = { + ...trace, + trace_id: 'e2e-trace-failed', + start: T0 - 120_000, + end: T0 - 119_000, + status: 'error', + errors: 1, + error_message: 'tool timed out', + output_preview: null, +}; + +const session: ObserveSessionRow = { + session_id: 'e2e-session', + agent_id: 'e2e-agent', + agent_name: 'Support bot', + status: 'active', + started_at: T0, + ended_at: T0 + 1_800, + traces: 1, + input_tokens: 120, + output_tokens: 48, + cost_usd: 0.0012, + last_input_preview: 'Where is my order?', +}; + +const observation = { + trace_id: 'e2e-trace', + session_id: 'e2e-session', + agent_id: 'e2e-agent', + status: 'ok', + model: null, + usage_input_tokens: null, + usage_output_tokens: null, + usage_cached_tokens: null, + usage_reasoning_tokens: null, + cost_usd: null, + cost_partial: null, + usage_estimated: null, + truncated: false, + attrs: null, +}; + +const score: ObserveScoreRow = { + score_id: 'e2e-score', + trace_id: 'e2e-trace', + obs_id: null, + name: 'e2e-eval', + value: 1, + source: 'eval', + comment: null, + agent_id: 'e2e-agent', + created_at: T0 + 2_000, +}; + +const traceDetail: ObserveTraceDetail = { + trace_id: 'e2e-trace', + observations: [ + { ...observation, obs_id: 'e2e-root', parent_obs_id: null, kind: 'agent', name: 'support turn', start: T0, end: T0 + 1_800, input: 'Where is my order?', output: 'It ships tomorrow.' }, + { + ...observation, + obs_id: 'e2e-obs', + parent_obs_id: 'e2e-root', + kind: 'generation', + name: 'llm', + start: T0 + 100, + end: T0 + 900, + input: '[{"role":"user","content":"Where is my order?"}]', + output: 'Let me look that up.', + model: 'mock-echo', + usage_input_tokens: 120, + usage_output_tokens: 48, + cost_usd: 0.0012, + }, + { ...observation, obs_id: 'e2e-tool', parent_obs_id: 'e2e-root', kind: 'tool', name: 'lookup_order', start: T0 + 1_000, end: T0 + 1_700, input: '{"id":"42"}', output: '{"eta":"tomorrow"}' }, + ], + scores: [score, { ...score, score_id: 'e2e-human', name: 'helpful', source: 'human', comment: 'clear answer', obs_id: null }], +}; + +const agent: ObserveAgentRow = { + agent_id: 'e2e-agent', + project: 'e2e', + name: 'Support bot', + version: '1.0.0', + first_seen: T0 - 86_400_000, + last_heartbeat: NOW - 5_000, + status: 'online', + heartbeat_age_ms: 5_000, + heartbeat_interval_ms: 15_000, + hostname: 'e2e-host', + pid: 4242, + dropped: 0, + runs_today: 12, + error_rate_today: 0.08, +}; + +const HOUR = 3_600_000; +const insights: ObserveInsights = { + window: { since: NOW - 24 * HOUR, until: NOW }, + agents: [ + { agent_id: 'e2e-agent', project: 'e2e', name: 'Support bot', runs: 12, runs_per_day: 12, errors: 1, error_rate: 0.08, cost_usd: 0.0144, input_tokens: 1_440, output_tokens: 576, p50_latency_ms: 1_200, p95_latency_ms: 2_400 }, + ], + series: ['runs', 'errors', 'cost_usd', 'input_tokens', 'output_tokens', 'p95_latency_ms', 'error_rate'].flatMap((metric, m) => + [0, 1, 2, 3].map((i) => ({ bucket_start: NOW - (4 - i) * HOUR, granularity: 'hour', agent_id: 'e2e-agent', metric, value: (i + 1) * (m + 1) })), + ), +}; + +const promptVersion = (version: number, labels: string[], text: string): ObservePromptVersion => ({ + prompt_id: `e2e-prompt-${version}`, + name: 'e2e-prompt', + version, + labels, + variables: ['customer'], + hash: `hash-${version}`, + text, + created_at: T0 - (3 - version) * HOUR, + uses: version, +}); +const promptVersions = [promptVersion(2, ['production'], 'Hello {{customer}}, how can I help today?'), promptVersion(1, [], 'Hello {{customer}}.')]; + +const hubEntry = (key: string, value: string, scope: ObserveHubEntry['scope'], ttl: number | null): ObserveHubEntry => ({ + entry_id: `e2e-hub-${key}`, + namespace: 'e2e-namespace', + agent_id: 'e2e-agent', + key, + value, + scope, + created_at: T0, + ttl, + expires_at: ttl === null ? null : T0 + ttl, +}); +const hubEntries = [hubEntry('order-policy', '{"returns_days":30}', 'fleet', null), hubEntry('last-customer', 'cust-42', 'private', 24 * HOUR)]; + +const json = (route: Route, body: unknown, status = 200) => + route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); + +/** Answer every `/observe/**` request from the fixtures above; anything unknown is the daemon's 404. */ +export async function stubObserve(page: Page): Promise { + await page.route('**/observe/**', (route) => { + const url = new URL(route.request().url()); + const path = url.pathname.replace(/^\/observe/, ''); + const q = url.searchParams; + if (path === '/health') return json(route, { service: 'lousho-observe', ok: true, version: 'e2e' }); + if (path === '/__target') return json(route, { url: 'http://127.0.0.1:7777' }); + if (path === '/api/v1/traces') { + const traces = q.get('status') === 'error' ? [failedTrace] : [trace, failedTrace]; + return json(route, { traces, total: traces.length, limit: 50, offset: 0 }); + } + if (path === '/api/v1/traces/e2e-trace') return json(route, traceDetail); + if (path === '/api/v1/sessions') return json(route, { sessions: [session], total: 1, limit: 50, offset: 0 }); + if (path === '/api/v1/sessions/e2e-session') return json(route, { session, traces: [trace] }); + if (path === '/api/v1/agents') return json(route, { agents: [agent] }); + if (path === '/api/v1/insights') return json(route, insights); + if (path === '/api/v1/prompts') return json(route, { prompts: [{ ...promptVersions[0], versions: 2, uses: 3 }] }); + if (path === '/api/v1/prompts/e2e-prompt') return json(route, { name: 'e2e-prompt', versions: promptVersions }); + if (path === '/api/v1/hub' && route.request().method() === 'GET') { + const ns = q.get('namespace'); + const scope = q.get('scope'); + return json(route, { entries: hubEntries.filter((e) => (!ns || e.namespace === ns) && (!scope || e.scope === scope)) }); + } + if (path === '/api/v1/scores') { + if (q.get('aggregate') === 'eval') + return json(route, { evals: [{ name: 'e2e-eval', count: 4, passed: 3, pass_rate: 0.75, avg_value: 0.75, last_ts: T0 + 2_000 }] }); + return json(route, { scores: [score, { ...score, score_id: 'e2e-score-2', value: 0, comment: 'wrong order id' }] }); + } + return json(route, { error: 'not found' }, 404); + }); +} diff --git a/apps/agent-forge/package.json b/apps/agent-forge/package.json index 74c209c1..5899d2d8 100644 --- a/apps/agent-forge/package.json +++ b/apps/agent-forge/package.json @@ -35,6 +35,7 @@ "devDependencies": { "@ai-sdk/anthropic": "^0.0.42", "@ai-sdk/openai": "^0.0.42", + "@axe-core/playwright": "4.13.0", "@playwright/test": "^1.63.0", "@types/dagre": "^0.7.52", "@types/express": "^5.0.5", diff --git a/apps/agent-forge/src/components/SettingsPanel.tsx b/apps/agent-forge/src/components/SettingsPanel.tsx index 8c9aa459..7f67d8c2 100644 --- a/apps/agent-forge/src/components/SettingsPanel.tsx +++ b/apps/agent-forge/src/components/SettingsPanel.tsx @@ -240,11 +240,13 @@ function ProfileEditor({ profileDraft, isActive, onChange, onSave, onActivate }:
Enable OTel tracing -
onChange({ ...profileDraft, otelEnabled: !profileDraft.otelEnabled })} role="switch" aria-checked={profileDraft.otelEnabled} + aria-label="Enable OTel tracing" />
diff --git a/apps/agent-forge/src/screens/__tests__/routes.test.tsx b/apps/agent-forge/src/screens/__tests__/routes.test.tsx index 660b09c6..e93da83c 100644 --- a/apps/agent-forge/src/screens/__tests__/routes.test.tsx +++ b/apps/agent-forge/src/screens/__tests__/routes.test.tsx @@ -20,6 +20,7 @@ const appState = vi.hoisted(() => ({ vi.mock('../../state/AppState', () => ({ useAppState: () => appState })); import { screenFor, useScreen } from '../routes'; +import { ROUTES } from '../routeTable'; import { Placeholder } from '../Placeholder'; import { Topbar } from '../../components/Topbar'; @@ -74,6 +75,12 @@ describe('screenFor', () => { expect(screenFor('#/')).toBeNull(); expect(screenFor('#/nope')).toBeNull(); }); + it("gives every route an example path that lands on that route's screen", () => { + for (const { pattern, screen, example } of ROUTES) { + expect(example.split('/').length, pattern).toBe(pattern.split('/').length); + expect(screenFor(`#${example}`), example).toBe(screen); + } + }); }); describe('useScreen', () => { diff --git a/apps/agent-forge/src/screens/agents/useLeaveAgentsGuard.ts b/apps/agent-forge/src/screens/agents/useLeaveAgentsGuard.ts index a689d113..8a250ae1 100644 --- a/apps/agent-forge/src/screens/agents/useLeaveAgentsGuard.ts +++ b/apps/agent-forge/src/screens/agents/useLeaveAgentsGuard.ts @@ -2,7 +2,7 @@ import { useLayoutEffect, useRef, useState } from 'react'; import { navigate } from '../../router/router'; import { useRoute } from '../../router/useRoute'; import { useAppState } from '../../state/AppState'; -import type { ScreenId } from '../routes'; +import type { ScreenId } from '../routeTable'; import { confirmDiscardChanges } from './unsavedChanges'; /** diff --git a/apps/agent-forge/src/screens/routeTable.ts b/apps/agent-forge/src/screens/routeTable.ts new file mode 100644 index 00000000..4dce771c --- /dev/null +++ b/apps/agent-forge/src/screens/routeTable.ts @@ -0,0 +1,44 @@ +/** The Studio's screens, one id per top-level view. */ +export type ScreenId = + | 'agents' + | 'playground' + | 'traces' + | 'trace' + | 'threads' + | 'monitoring' + | 'scores' + | 'evals' + | 'prompts' + | 'hub' + | 'settings'; + +/** One hash route: its pattern, the screen it shows, and a concrete path that exercises it. */ +export interface RouteEntry { + readonly pattern: string; + readonly screen: ScreenId; + /** A real hash path (params filled in) for this pattern - the axe e2e scans every one. */ + readonly example: string; +} + +/** + * Hash route patterns, first match wins. Deep-link params are accepted now and read by the screens that land later. + * Kept free of React so the Playwright specs can import it. + */ +export const ROUTES: readonly RouteEntry[] = [ + { pattern: '/agents', screen: 'agents', example: '/agents' }, + { pattern: '/agents/:id', screen: 'agents', example: '/agents/e2e-agent' }, + { pattern: '/playground', screen: 'playground', example: '/playground' }, + { pattern: '/traces', screen: 'traces', example: '/traces' }, + { pattern: '/traces/:traceId', screen: 'trace', example: '/traces/e2e-trace' }, + { pattern: '/threads', screen: 'threads', example: '/threads' }, + { pattern: '/threads/:sessionId', screen: 'threads', example: '/threads/e2e-session' }, + { pattern: '/monitoring', screen: 'monitoring', example: '/monitoring' }, + { pattern: '/evals/scores', screen: 'scores', example: '/evals/scores' }, + { pattern: '/evals/:section', screen: 'evals', example: '/evals/datasets' }, + { pattern: '/evals/:section/:id', screen: 'evals', example: '/evals/datasets/e2e-dataset' }, + { pattern: '/prompts', screen: 'prompts', example: '/prompts' }, + { pattern: '/prompts/:name', screen: 'prompts', example: '/prompts/e2e-prompt' }, + { pattern: '/hub', screen: 'hub', example: '/hub' }, + { pattern: '/hub/:namespace', screen: 'hub', example: '/hub/e2e-namespace' }, + { pattern: '/settings', screen: 'settings', example: '/settings' }, +]; diff --git a/apps/agent-forge/src/screens/routes.tsx b/apps/agent-forge/src/screens/routes.tsx index 61c76ac6..4a65017f 100644 --- a/apps/agent-forge/src/screens/routes.tsx +++ b/apps/agent-forge/src/screens/routes.tsx @@ -14,39 +14,7 @@ import { ScoresScreen } from './scores/ScoresScreen'; import { PlaygroundRoute } from './playground/PlaygroundScreen'; import { useLeaveAgentsGuard } from './agents/useLeaveAgentsGuard'; import { Placeholder } from './Placeholder'; - -export type ScreenId = - | 'agents' - | 'playground' - | 'traces' - | 'trace' - | 'threads' - | 'monitoring' - | 'scores' - | 'evals' - | 'prompts' - | 'hub' - | 'settings'; - -/** Hash route patterns, first match wins. Deep-link params are accepted now and read by the screens that land later. */ -const ROUTES: { pattern: string; screen: ScreenId }[] = [ - { pattern: '/agents', screen: 'agents' }, - { pattern: '/agents/:id', screen: 'agents' }, - { pattern: '/playground', screen: 'playground' }, - { pattern: '/traces', screen: 'traces' }, - { pattern: '/traces/:traceId', screen: 'trace' }, - { pattern: '/threads', screen: 'threads' }, - { pattern: '/threads/:sessionId', screen: 'threads' }, - { pattern: '/monitoring', screen: 'monitoring' }, - { pattern: '/evals/scores', screen: 'scores' }, - { pattern: '/evals/:section', screen: 'evals' }, - { pattern: '/evals/:section/:id', screen: 'evals' }, - { pattern: '/prompts', screen: 'prompts' }, - { pattern: '/prompts/:name', screen: 'prompts' }, - { pattern: '/hub', screen: 'hub' }, - { pattern: '/hub/:namespace', screen: 'hub' }, - { pattern: '/settings', screen: 'settings' }, -]; +import { ROUTES, type ScreenId } from './routeTable'; /** The screen a hash shows, or `null` for an empty hash / unknown route. */ export function screenFor(hash: string): ScreenId | null { diff --git a/package-lock.json b/package-lock.json index 3aa65449..cb252e42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -70,6 +70,7 @@ "@opentelemetry/api": "^1.9.1", "ai": "^4.3.19 || ^6.0.0 || ^7.0.0", "dockerode": "^5.0.1", + "isolated-vm": "^6.2.0 || ^7.0.0", "ollama-ai-provider": "^1.2.0", "ollama-ai-provider-v2": "^2.0.0 || ^3.0.0 || ^4.0.0", "prompts": "^2.4.2", @@ -104,6 +105,9 @@ "dockerode": { "optional": true }, + "isolated-vm": { + "optional": true + }, "ollama-ai-provider": { "optional": true }, @@ -145,6 +149,7 @@ "devDependencies": { "@ai-sdk/anthropic": "^0.0.42", "@ai-sdk/openai": "^0.0.42", + "@axe-core/playwright": "4.13.0", "@playwright/test": "^1.63.0", "@types/dagre": "^0.7.52", "@types/express": "^5.0.5", @@ -876,6 +881,19 @@ "node": ">=18.0.0" } }, + "node_modules/@axe-core/playwright": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.13.0.tgz", + "integrity": "sha512-6YLx+kxXu5GJceG4ozFg+33a2EMTdjYwWGloJ3sb9Kta5pp+ZNS53uxGVog5JetIY8s++P5UrtX+cri+u0VAVg==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.13.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -7144,6 +7162,16 @@ "dev": true, "license": "MIT" }, + "node_modules/axe-core": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", + "integrity": "sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",