From 71ef542cb222b8149f4224f1173ccdc86e2da8e4 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 29 Oct 2026 11:22:31 +0300 Subject: [PATCH 1/4] feat(studio): Evaluators card grid and a sectioned evaluator form with a sticky footer The rule list is a card grid (kind icon tile, name link stretched over the card, enabled switch outside the link, target filter, sample rate, 24h runs and last run), with Kind/State filters and TablePager paging; New evaluator moves to the header. The form is General / Logic / Sampling cards (kind as radio cards, code and rubric in a mono editor surface) over a sticky footer; the edit view gets breadcrumbs and Test on trace as a side card. The shared Evaluations page and card-grid rules move from datasets.css to evals.css, and the grid skeleton is shared with Datasets. Co-Authored-By: Claude Opus 5.5 --- .../src/screens/datasets/DatasetsScreen.tsx | 25 +- .../src/screens/datasets/datasets.css | 187 +------ .../src/screens/evals/GridSkeleton.tsx | 26 + apps/agent-forge/src/screens/evals/evals.css | 148 +++++- .../src/screens/evaluators/EvaluatorCard.tsx | 96 ++++ .../screens/evaluators/EvaluatorDetail.tsx | 81 ++- .../src/screens/evaluators/EvaluatorForm.tsx | 432 ++++++++++----- .../screens/evaluators/EvaluatorsScreen.tsx | 282 ++++++---- .../__tests__/EvaluatorsScreen.test.tsx | 46 +- .../src/screens/evaluators/evaluators.css | 500 ++++++++++++++++-- 10 files changed, 1308 insertions(+), 515 deletions(-) create mode 100644 apps/agent-forge/src/screens/evals/GridSkeleton.tsx create mode 100644 apps/agent-forge/src/screens/evaluators/EvaluatorCard.tsx diff --git a/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx b/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx index 0510356c3..cb8d82136 100644 --- a/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx +++ b/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx @@ -19,9 +19,9 @@ import { fmtInt } from '../../observe/format'; import { Card } from '../../ui/Card'; import { TablePager } from '../../ui/DataTable'; import { EmptyState } from '../../ui/EmptyState'; -import { Skeleton } from '../../ui/Skeleton'; import { useToast } from '../../ui/Toast'; import { EvalsHeader } from '../evals/EvalsTabs'; +import { GridSkeleton } from '../evals/GridSkeleton'; import { DatasetDetail } from './DatasetDetail'; import { DATASETS_PAGE_SIZE, datasetPath, datasetsPath, parseOffset } from './datasetsUrl'; @@ -149,27 +149,6 @@ function DatasetCard({ dataset: d }: { dataset: ObserveDataset }) { ); } -/** The grid's first-load placeholder: six skeleton cards and one status line. */ -function GridSkeleton() { - return ( -
- - Loading datasets - - -
- ); -} - function NewDatasetButton({ open, onClick }: { open: boolean; onClick: () => void }) { return ( @@ -65,7 +80,7 @@ function TestOnTrace({ rule, prefill }: { rule: ObserveEvaluator; prefill: strin

)} {result && ( -
+
Value
{result.value === null ? 'null' : String(result.value)}
Comment
@@ -106,7 +121,7 @@ function EditRule({ rule, back, onSaved }: { rule: ObserveEvaluator; back: strin }; return ( - <> +
void remove()}> + } /> - +
); } +/** The edit view while the rule loads: a card of skeleton lines. */ +const LoadingRule = () => ( + + + +); + /** `#/evals/evaluators/:id`; an unknown rule shows an empty state with the way back. */ export function EvaluatorDetail({ ruleId }: { ruleId: string }) { const { query } = useRoute(); const [refreshKey, setRefreshKey] = useState(0); const rule = useEvaluator(ruleId, refreshKey); const back = evaluatorsPath(parseListState(query)); - const backLink = All evaluators; - if (useDaemonDown(false, [rule])) return ; - let body; - if (rule.loading && !rule.data) body =

Loading evaluator...

; + const down = useDaemonDown(false, [rule]); + const name = rule.data?.name ?? ruleId; + let body: ReactNode; + if (down) body = ; + else if (rule.loading && !rule.data) body = ; else if (rule.error) body = ; - else if (!rule.data) body = ; + else if (!rule.data) + body = ( + + + All evaluators + + } + /> + + ); else body = setRefreshKey((k) => k + 1)} />; return ( <> - -

Evaluator {rule.data ? {rule.data.name} : null}

+ +

Evaluator {rule.data?.name}

{body} ); diff --git a/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx index 4081377e9..4b6ea627f 100644 --- a/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx +++ b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx @@ -1,18 +1,21 @@ /** - * The evaluator form the list's "New evaluator" and the edit view share: - * name, kind (`code` with a function textarea, or `judge` with a model and a - * rubric), the trace filter (agent, status, name contains), the sample rate - * and the enabled flag. A judge needs a provider key, stored in Studio's - * settings or set as the provider's env var where Studio runs (judges run - * with either): without one the judge option is disabled, with a hint that - * links Settings. - * A failed save (a taken name is the daemon's 409) shows inline. + * The evaluator form the list's "New evaluator" and the edit view share, one + * card per section: General (name, enabled, kind), Logic (`code`: the function + * in a mono editor; `judge`: a model and the rubric) and Sampling (the trace + * filter - agent, status, name contains - and the sample rate), over a footer + * that sticks to the bottom of the screen with the save actions. A judge needs + * a provider key, stored in Studio's settings or set as the provider's env var + * where Studio runs (judges run with either): without one the judge option is + * disabled, with a hint that links Settings. + * A failed save (a taken name is the daemon's 409) shows inline in the footer. */ -import { useState, type FormEvent, type ReactNode } from 'react'; +import { useId, useState, type FormEvent, type ReactNode } from 'react'; import type { ProviderKeyStatus } from '../../../shared/wireTypes'; import { ObserveApiError } from '../../observe/observeClient'; import type { NewEvaluator, ObserveAgentRow } from '../../observe/types'; +import { Card } from '../../ui/Card'; import { MOCK_PROVIDER, defaultProvider, providerOptions } from '../playground/request'; +import { KIND_ICON } from './EvaluatorCard'; import { draftProblem, toNewEvaluator, type EvaluatorDraft } from './evaluatorDraft'; type SetField = (field: K, value: EvaluatorDraft[K]) => void; @@ -26,6 +29,54 @@ const saveError = (error: unknown): string => /** The providers a judge can run on: a key stored in Settings or the provider's env var (`envKey`) both count. */ const judgeKeys = (keys: readonly ProviderKeyStatus[]): ProviderKeyStatus[] => keys.map((k) => ({ ...k, hasKey: k.hasKey || k.envKey })); +const KINDS: { kind: EvaluatorDraft['kind']; title: string; description: string }[] = [ + { kind: 'code', title: 'Code', description: 'A JavaScript function of the trace returns the score.' }, + { kind: 'judge', title: 'LLM judge', description: 'A model grades the trace against your rubric.' }, +]; + +/** One kind as a radio card: the native radio, the kind icon, its title (the radio's name) and a line on what it does. */ +function KindOption({ + option, + checked, + disabled, + describedBy, + onChoose, +}: { + option: (typeof KINDS)[number]; + checked: boolean; + disabled: boolean; + describedBy?: string; + onChoose: () => void; +}) { + const id = useId(); + const Icon = KIND_ICON[option.kind]; + return ( + + ); +} + function KindField({ draft, choose, @@ -36,26 +87,25 @@ function KindField({ judgeAllowed: boolean; }) { return ( -
- Kind - - +
+ Kind +
+ {KINDS.map((option) => { + const blocked = option.kind === 'judge' && !judgeAllowed; + return ( + choose(option.kind)} + /> + ); + })} +
{!judgeAllowed && ( -

+

Add a provider key in Settings to use judge evaluators.

)} @@ -63,36 +113,83 @@ function KindField({ ); } -function FilterFields({ draft, set, agents }: { draft: EvaluatorDraft; set: SetField; agents: ObserveAgentRow[] }) { - const known = agents.some((a) => a.agent_id === draft.agentId); +function GeneralCard({ + draft, + set, + choose, + judgeAllowed, +}: { + draft: EvaluatorDraft; + set: SetField; + choose: (kind: EvaluatorDraft['kind']) => void; + judgeAllowed: boolean; +}) { return ( -
- Runs on traces where - - - -
+ +
+
+ + +
+ +
+ + ); +} + +/** A labelled mono editor surface: the label is its header bar, `tag` a decorative language note. */ +function Editor({ + label, + tag, + name, + value, + placeholder, + onChange, +}: { + label: ReactNode; + tag: string; + name: string; + value: string; + placeholder?: string; + onChange: (value: string) => void; +}) { + return ( +