feat(studio): Evaluators card grid and a sectioned evaluator form with a sticky footer

apps/agent-forge/e2e/a11y.spec.ts

@@ -140,6 +140,8 @@ const ITEM_H1: Record<string, string> = {
   '/prompts/e2e-prompt?diff=1,2': 'e2e-prompt',
   '/prompts/:name': 'e2e-prompt',
   '/hub/:namespace': 'e2e-namespace',
+  '/evals/queues/:id': 'support-review',
+  '/evals/evaluators/:id': 'answered',
 };
 for (const { label, path } of SCANS) {
   test(`${label} keeps the landmarks and passes axe`, async ({ page }) => {
@@ -246,7 +248,7 @@ test('the axe scans see the stubbed daemon data, not the down state', async ({ p
     ['/hub', 'e2e-namespace'],
     ['/hub/e2e-namespace', 'order-policy'],
     ['/evals/queues', 'support-review'],
-    ['/evals/queues/e2e-queue', 'lookup_order'],
+    ['/evals/queues/e2e-queue', 'Where is my order?'],
   ]) {
     await page.goto(`/?token=e2e-studio-token#${path}`);
     await expect(main.getByText(text).first(), path).toBeVisible();

apps/agent-forge/e2e/docShots.spec.ts

@@ -393,7 +393,7 @@ test('datasets: one dataset with its examples', async ({ page }) => {
 
 test('evaluators: the rule list', async ({ page }) => {
   await open(page, '/evals/evaluators');
-  await expect(rows(page)).toHaveCount(2);
+  await expect(page.getByRole('main').locator('.ev-card')).toHaveCount(2);
   await expect(page.getByRole('switch', { name: 'answered enabled' })).toBeChecked();
   await shoot(page, 'evaluators');
 });
@@ -1214,3 +1214,140 @@ test('x-threads: detail at 375 wide', async ({ page }) => {
   await shoot(page, 'x-thread-narrow');
   await page.screenshot({ path: path.join(OUT, 'studio-x-thread-narrow-full.png'), fullPage: true });
 });
+
+// --- Evaluators and annotation queues (x- shots, P7) -------------------------
+
+const RULES = [
+  { rule_id: 'e2e-evaluator', name: 'answered', enabled: true, filter: { agent_id: 'e2e-agent', status: 'ok' }, sample_rate: 1, kind: 'code', config: { source: '(trace) => (trace.output ? 1 : 0)' } },
+  { rule_id: 'e2e-judge', name: 'polite', enabled: false, filter: {}, sample_rate: 0.5, kind: 'judge', config: { model: 'openai/gpt-4o-mini', rubric: 'Is the reply polite? Score 1 when it is courteous and calm, 0 otherwise.' } },
+  { rule_id: 'x-refund', name: 'refund-policy-cited', enabled: true, filter: { name_contains: 'refund' }, sample_rate: 0.25, kind: 'judge', config: { model: 'anthropic/claude-haiku', rubric: 'Does the reply cite the refund policy?' } },
+  { rule_id: 'x-tools', name: 'tool-calls-under-budget-for-every-order-lookup-in-the-support-fleet', enabled: true, filter: { status: 'error' }, sample_rate: 0.1, kind: 'code', config: { source: '(trace) => trace.observations.length < 12 ? 1 : 0' } },
+].map((r) => ({ created_at: NOW - 30 * HOUR, updated_at: NOW - 30 * HOUR, ...r }));
+
+const QUEUES = [
+  { queue_id: 'e2e-queue', name: 'support-review', rubric: [{ name: 'helpfulness', min: 0, max: 5 }, { name: 'tone', min: 1, max: 3 }], counts: { pending: 1, done: 2, skipped: 0 } },
+  { queue_id: 'x-refunds', name: 'refund-escalations', rubric: [{ name: 'correct', min: 0, max: 1 }, { name: 'policy cited', min: 0, max: 1 }, { name: 'empathy', min: 1, max: 5 }], counts: { pending: 14, done: 41, skipped: 3 } },
+  { queue_id: 'x-new', name: 'multilingual-greetings-weekly-spot-check', rubric: [{ name: 'fluency' }], counts: { pending: 25, done: 0, skipped: 0 } },
+  { queue_id: 'x-done', name: 'october-regressions', rubric: [{ name: 'fixed', min: 0, max: 1 }], counts: { pending: 0, done: 57, skipped: 4 } },
+].map((q, i) => ({ created_at: NOW - (i + 1) * 26 * HOUR, ...q }));
+
+async function evaluatorList(page: Page, list: unknown[]) {
+  await page.route('**/observe/api/v1/evaluators?**', (route) =>
+    route.request().method() === 'GET' ? json(route, { evaluators: list, total: list.length, limit: 50, offset: 0 }) : route.fallback(),
+  );
+}
+
+async function queueList(page: Page, list: unknown[]) {
+  await page.route('**/observe/api/v1/queues?**', (route) =>
+    route.request().method() === 'GET' ? json(route, { queues: list, total: list.length, limit: 50, offset: 0 }) : route.fallback(),
+  );
+}
+
+const main = (page: Page) => page.getByRole('main');
+
+test('evals x: evaluators grid', async ({ page }) => {
+  await evaluatorList(page, RULES);
+  await open(page, '/evals/evaluators');
+  await expect(main(page).getByRole('switch', { name: 'polite enabled' })).toBeVisible();
+  await shoot(page, 'x-evals-evaluators-grid');
+});
+
+test('evals x: evaluators empty', async ({ page }) => {
+  await evaluatorList(page, []);
+  await open(page, '/evals/evaluators');
+  await expect(main(page).locator('.empty-state')).toBeVisible();
+  await shoot(page, 'x-evals-evaluators-empty');
+});
+
+test('evals x: new evaluator form (code rule)', async ({ page }) => {
+  await open(page, '/evals/evaluators');
+  await page.getByRole('button', { name: 'New evaluator' }).first().click();
+  await expect(page.getByRole('form', { name: 'New evaluator' })).toBeVisible();
+  await shoot(page, 'x-evals-evaluator-new-code');
+});
+
+test('evals x: new evaluator form (LLM judge)', async ({ page }) => {
+  // A stored provider key enables the judge option (no real key is sent anywhere: the route answers in the browser).
+  await page.route('**/settings/providers', (route) =>
+    route.request().method() === 'GET'
+      ? json(route, [
+          { provider: 'openai', hasKey: true, masked: '<your-key>', envKey: false },
+          { provider: 'anthropic', hasKey: false, masked: null, envKey: false },
+        ])
+      : route.fallback(),
+  );
+  await open(page, '/evals/evaluators');
+  await page.getByRole('button', { name: 'New evaluator' }).first().click();
+  const form = page.getByRole('form', { name: 'New evaluator' });
+  await form.getByRole('radio', { name: 'LLM judge' }).check();
+  await form.locator('[name="evaluator-name"]').fill('helpful');
+  await form.locator('[name="evaluator-model"]').fill('gpt-4o-mini');
+  await form.locator('[name="evaluator-rubric"]').fill('Score 1 when the reply answers the customer question with the order status, 0 otherwise.');
+  await shoot(page, 'x-evals-evaluator-new-judge');
+});
+
+test('evals x: evaluator edit view (code rule)', async ({ page }) => {
+  await open(page, '/evals/evaluators/e2e-evaluator');
+  await expect(page.getByRole('form', { name: 'Edit evaluator' })).toBeVisible();
+  await shoot(page, 'x-evals-evaluator-edit-code');
+});
+
+test('evals x: evaluator edit view (LLM judge)', async ({ page }) => {
+  await page.route('**/observe/api/v1/evaluators/e2e-judge', (route) =>
+    route.request().method() === 'GET' ? json(route, { evaluator: RULES[1] }) : route.fallback(),
+  );
+  await open(page, '/evals/evaluators/e2e-judge');
+  await expect(page.locator('[name="evaluator-rubric"]')).toBeVisible();
+  await shoot(page, 'x-evals-evaluator-edit-judge');
+});
+
+test('evals x: queues list', async ({ page }) => {
+  await queueList(page, QUEUES);
+  await open(page, '/evals/queues');
+  await expect(main(page).getByText('refund-escalations').first()).toBeVisible();
+  await shoot(page, 'x-evals-queues-list');
+});
+
+test('evals x: queues empty', async ({ page }) => {
+  await queueList(page, []);
+  await open(page, '/evals/queues');
+  await expect(main(page).locator('.empty-state')).toBeVisible();
+  await shoot(page, 'x-evals-queues-empty');
+});
+
+test('evals x: new queue form with rubric rows', async ({ page }) => {
+  await queueList(page, QUEUES);
+  await open(page, '/evals/queues');
+  const toggle = page.getByRole('button', { name: 'New queue' });
+  if (await toggle.count()) await toggle.first().click();
+  const form = page.getByRole('form', { name: 'New annotation queue' });
+  await form.locator('[name="queue-name"]').fill('billing-review');
+  await form.locator('[name="rubric-name-0"]').fill('correct');
+  await form.locator('[name="rubric-min-0"]').fill('0');
+  await form.locator('[name="rubric-max-0"]').fill('1');
+  await form.getByRole('button', { name: /Add rubric (row|entry)/ }).click();
+  await form.locator('[name="rubric-name-1"]').fill('empathy');
+  await form.locator('[name="rubric-description-1"]').fill('Did the reply acknowledge the problem?');
+  await form.locator('[name="rubric-min-1"]').fill('1');
+  await form.locator('[name="rubric-max-1"]').fill('5');
+  await shoot(page, 'x-evals-queues-new');
+});
+
+test('evals x: queue review at 375 wide', async ({ page }) => {
+  await page.setViewportSize({ width: 375, height: 800 });
+  await open(page, '/evals/queues/e2e-queue');
+  await expect(page.locator('[data-review-trace="e2e-trace"]')).toBeVisible();
+  await shoot(page, 'x-evals-queue-review-375');
+  await page.getByRole('form', { name: 'Scores' }).getByRole('button', { name: 'Skip' }).scrollIntoViewIfNeeded();
+  await shoot(page, 'x-evals-queue-review-375-rubric');
+});
+
+test('evals x: queue review, queue is empty', async ({ page }) => {
+  await page.route('**/observe/api/v1/queues/e2e-queue/next', (route) => route.fulfill({ status: 204 }));
+  await page.route('**/observe/api/v1/queues/e2e-queue', (route) =>
+    json(route, { queue: { ...QUEUES[0], counts: { pending: 0, done: 2, skipped: 1 } } }),
+  );
+  await open(page, '/evals/queues/e2e-queue');
+  await expect(main(page).getByText('Queue is empty')).toBeVisible();
+  await shoot(page, 'x-evals-queue-empty');
+});

apps/agent-forge/e2e/revamp.spec.ts

@@ -208,9 +208,11 @@ const CONTROLS: Record<string, (main: Locator, page: Page) => Promise<void>> = {
     await expect(main.getByRole('combobox', { name: 'State' })).toHaveValue('true');
     await expect(main.getByRole('switch', { name: 'answered enabled' })).toBeChecked();
   },
-  '/evals/evaluators/:id': async (main) => {
+  '/evals/evaluators/:id': async (main, page) => {
     await expect(main.getByRole('textbox', { name: 'Trace id' })).toHaveValue('e2e-trace');
-    await expect(main.getByRole('link', { name: 'All evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true');
+    // The way back is the header breadcrumb (the screen header sits outside <main>).
+    const crumbs = page.getByRole('navigation', { name: 'Breadcrumb' });
+    await expect(crumbs.getByRole('link', { name: 'Evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true');
   },
   '/prompts/:name': async (main) => {
     await expect(main.getByRole('combobox', { name: 'Compare version A' })).toHaveValue('1');
@@ -323,6 +325,8 @@ test('queue review at 1280px: a trace tree row name never wraps one letter per l
   await page.setViewportSize({ width: 1280, height: 800 });
   await page.goto(`${APP}#/evals/queues/e2e-queue`);
   await expect(page.locator('[data-review-trace="e2e-trace"]')).toBeVisible();
+  // The review opens on the root step's messages; the tree is the Trace view.
+  await page.getByRole('group', { name: 'Item view' }).getByRole('button', { name: 'Trace' }).click();
   const names = page.locator('[data-review-trace="e2e-trace"] .run-name');
   await expect(names.first()).toBeVisible();
   for (const name of await names.all()) {

apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx

@@ -19,9 +19,9 @@ import { fmtInt } from '../../observe/format';
 import { Card } from '../../ui/Card';
 import { TablePager } from '../../ui/DataTable';
 import { EmptyState } from '../../ui/EmptyState';
-import { Skeleton } from '../../ui/Skeleton';
 import { useToast } from '../../ui/Toast';
 import { EvalsHeader } from '../evals/EvalsTabs';
+import { GridSkeleton } from '../evals/GridSkeleton';
 import { DatasetDetail } from './DatasetDetail';
 import { DATASETS_PAGE_SIZE, datasetPath, datasetsPath, parseOffset } from './datasetsUrl';
 
@@ -149,27 +149,6 @@ function DatasetCard({ dataset: d }: { dataset: ObserveDataset }) {
   );
 }
 
-/** The grid's first-load placeholder: six skeleton cards and one status line. */
-function GridSkeleton() {
-  return (
-    <div className="ds-grid-loading">
-      <span role="status" className="visually-hidden">
-        Loading datasets
-      </span>
-      <ul className="ds-grid" aria-hidden="true">
-        {Array.from({ length: 6 }, (_, i) => (
-          <li key={i} className="ds-card ds-card-skeleton">
-            <Skeleton width={28} height={28} radius="var(--radius)" />
-            <Skeleton width="55%" height={12} />
-            <Skeleton width="85%" height={10} />
-            <Skeleton width="40%" height={10} />
-          </li>
-        ))}
-      </ul>
-    </div>
-  );
-}
-
 function NewDatasetButton({ open, onClick }: { open: boolean; onClick: () => void }) {
   return (
     <button type="button" className="btn btn-primary" aria-expanded={open} aria-controls={open ? FORM_ID : undefined} onClick={onClick}>
@@ -263,7 +242,7 @@ function DatasetList() {
         />
       )}
       <ObserveError error={list.error} />
-      {list.loading && !list.data && <GridSkeleton />}
+      {list.loading && !list.data && <GridSkeleton label="Loading datasets" />}
       {list.data && (
         <DatasetGrid datasets={list.data.datasets} total={list.data.total} offset={offset} creating={creating} onNew={() => setCreating(true)} />
       )}