diff --git a/CHANGELOG.md b/CHANGELOG.md index 15548ea3b..a4d45c921 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ opt-out. - Insights runs/day averages over the days an agent was active in the window, so an agent first seen today with 12 runs reports 12/day instead of 12/7. - An open run's root start is kept when the reporter's buffer trims or a crash flush cuts to its newest 16 MB, so a run whose child spans overflow the buffer is still listed. - The run inspector shows `error`, not `running`, for a failed step that has no end time. +- Studio trace viewer: an observation the crash sweep marked `interrupted` (or an open `error` one) now shows as terminal instead of "running" forever; its duration and the waterfall stop at the latest known end and the header clock stops ticking. - The observe daemon refuses cross-site browser writes and DNS rebinding on `/api/*`: a loopback-bound daemon answers only loopback `Host` names (403), writes carrying a non-loopback `Origin` (including `null`) or `Sec-Fetch-Site: cross-site` get 403, and a write with a body must be `Content-Type: application/json` (415) — `curl -d` needs `-H "Content-Type: application/json"`. The SDK reporter, CLI and Studio already comply. See [Observability](docs/observability.md). - Observe retention (`lousho observe start --retention-days`, `lousho observe prune`) also deletes old traces the sweep marked `interrupted` (a crashed run whose end never arrived); their age counts from when they were interrupted. diff --git a/apps/agent-forge/src/observe/__tests__/tree.test.ts b/apps/agent-forge/src/observe/__tests__/tree.test.ts index 97d1090f5..4b8524b9c 100644 --- a/apps/agent-forge/src/observe/__tests__/tree.test.ts +++ b/apps/agent-forge/src/observe/__tests__/tree.test.ts @@ -92,6 +92,12 @@ describe('traceBounds', () => { expect(bounds).toEqual({ minStart: 100, maxEnd: 1000 }); }); + it('does not count an open interrupted or error row to now', () => { + expect( + traceBounds([row({ start: 100, end: 400 }), row({ start: 200, end: null, status: 'interrupted' }), row({ start: 300, end: null, status: 'error' })], 1000), + ).toEqual({ minStart: 100, maxEnd: 400 }); + }); + it('is degenerate-safe on empty input', () => { expect(traceBounds([])).toEqual({ minStart: 0, maxEnd: 1 }); }); diff --git a/apps/agent-forge/src/observe/tree.ts b/apps/agent-forge/src/observe/tree.ts index ff73d2995..40eb84c65 100644 --- a/apps/agent-forge/src/observe/tree.ts +++ b/apps/agent-forge/src/observe/tree.ts @@ -75,14 +75,27 @@ export function flattenObservationTree(roots: ObservationNode[]): ObservationNod return flat; } -/** `[start, end]` bounds over every observation (an unfinished row counts to `now`). */ +/** The latest `end` among a node and its descendants, or `undefined` when none has ended. */ +export function latestKnownEnd(node: ObservationNode, seen = new Set()): number | undefined { + if (seen.has(node.row.obs_id)) return undefined; + seen.add(node.row.obs_id); + let latest = node.row.end ?? undefined; + for (const child of node.children) { + const end = latestKnownEnd(child, seen); + if (end !== undefined && (latest === undefined || end > latest)) latest = end; + } + return latest; +} + +/** `[start, end]` bounds over every observation (a running row counts to `now`; an open interrupted/error row stops at its start). */ export function traceBounds(rows: ObserveObservationRow[], now = Date.now()): { minStart: number; maxEnd: number } { if (rows.length === 0) return { minStart: 0, maxEnd: 1 }; let minStart = Infinity; let maxEnd = -Infinity; for (const row of rows) { minStart = Math.min(minStart, row.start); - maxEnd = Math.max(maxEnd, row.end ?? now); + const open = row.status === 'interrupted' || row.status === 'error' ? row.start : now; + maxEnd = Math.max(maxEnd, row.end ?? open); } return { minStart, maxEnd: Math.max(maxEnd, minStart + 1) }; } diff --git a/apps/agent-forge/src/screens/queues/QueueReview.tsx b/apps/agent-forge/src/screens/queues/QueueReview.tsx index e3352d472..f2127e238 100644 --- a/apps/agent-forge/src/screens/queues/QueueReview.tsx +++ b/apps/agent-forge/src/screens/queues/QueueReview.tsx @@ -13,7 +13,7 @@ import type { ObserveQueue, ObserveQueueItem } from '../../observe/types'; import { useDaemonDown, useNextQueueItem, useQueue, useTraceDetail } from '../../observe/useObserve'; import { EmptyState } from '../../ui/EmptyState'; import { TraceViewer, type TraceViewMode } from '../../ui/run/TraceViewer'; -import { displayName } from '../../ui/run/rows'; +import { displayName, isRunning } from '../../ui/run/rows'; import { useNow } from '../../ui/run/useNow'; import { fmtInt } from '../../observe/format'; import { RubricForm } from './RubricForm'; @@ -38,7 +38,7 @@ function ReviewItem({ queue, item, onDone }: { queue: ObserveQueue; item: Observ const [mode, setMode] = useState('tree'); const observations = trace.data?.observations; const root = useMemo(() => (observations ? nestObservations(observations)[0]?.row : undefined), [observations]); - const now = useNow(root?.end === null); + const now = useNow(root !== undefined && isRunning(root)); let body; if (down) body = ; diff --git a/apps/agent-forge/src/screens/traces/TraceHeader.tsx b/apps/agent-forge/src/screens/traces/TraceHeader.tsx index b9f90257b..16e7a69ea 100644 --- a/apps/agent-forge/src/screens/traces/TraceHeader.tsx +++ b/apps/agent-forge/src/screens/traces/TraceHeader.tsx @@ -6,6 +6,13 @@ import { displayName, displayStatus, elapsedMs } from '../../ui/run/rows'; import { AddToDataset } from './AddToDataset'; import { AddToQueue } from './AddToQueue'; +/** The latest `end` over every observation, or `undefined` when none has ended. */ +function latestEnd(rows: ObserveObservationRow[]): number | undefined { + let latest: number | undefined; + for (const row of rows) if (row.end !== null && (latest === undefined || row.end > latest)) latest = row.end; + return latest; +} + /** Trace-wide totals, summed over every observation the way the daemon's trace list does. */ function traceTotals(rows: ObserveObservationRow[]): { tokens: number | null; cost: number | null } { let tokens: number | null = null; @@ -88,7 +95,7 @@ export function TraceHeader({ )} {model ? {model} : '-'} {fmtDateTime(root.start)} - {fmtDuration(elapsedMs(root, now))} + {fmtDuration(elapsedMs(root, now, latestEnd(observations)))} {fmtInt(tokens)} {fmtCost(cost)} {root.session_id && ( diff --git a/apps/agent-forge/src/screens/traces/TraceScreen.tsx b/apps/agent-forge/src/screens/traces/TraceScreen.tsx index 1b1be08e7..f657e6a3e 100644 --- a/apps/agent-forge/src/screens/traces/TraceScreen.tsx +++ b/apps/agent-forge/src/screens/traces/TraceScreen.tsx @@ -9,6 +9,7 @@ import { useQueryState } from '../../router/useRoute'; import { EmptyState } from '../../ui/EmptyState'; import { tracesListPath } from './tracesUrl'; import { TraceViewer, type TraceViewMode } from '../../ui/run/TraceViewer'; +import { isRunning } from '../../ui/run/rows'; import { useNow } from '../../ui/run/useNow'; import { TraceFeedback } from './TraceFeedback'; import { TraceHeader } from './TraceHeader'; @@ -49,7 +50,7 @@ function useTrace(traceId: string, pollMs: number) { const observations = detail.data?.observations; const root = useMemo(() => (observations ? rootOf(observations) : undefined), [observations]); - const now = useNow(root?.end === null); + const now = useNow(root !== undefined && isRunning(root)); const agentName = agents.data?.find((a) => a.agent_id === root?.agent_id)?.name ?? undefined; return { detail, down, root, now, agentName }; } diff --git a/apps/agent-forge/src/screens/traces/__tests__/TraceScreen.test.tsx b/apps/agent-forge/src/screens/traces/__tests__/TraceScreen.test.tsx index 170987eec..1d210c820 100644 --- a/apps/agent-forge/src/screens/traces/__tests__/TraceScreen.test.tsx +++ b/apps/agent-forge/src/screens/traces/__tests__/TraceScreen.test.tsx @@ -144,6 +144,24 @@ describe('TraceScreen', () => { expect(header.querySelector('button[aria-label="Copy link"]')).not.toBeNull(); }); + it('header duration for an interrupted root with no end stops at the latest child end', async () => { + daemon({ + trace: { + trace_id: 't1', + observations: [ + obs({ obs_id: 'root', name: 'crashed run', start: T0, end: null, status: 'interrupted' }), + obs({ obs_id: 'c1', parent_obs_id: 'root', start: T0 + 100, end: T0 + 4000 }), + ], + scores: [], + }, + }); + const c = mount(); + await waitFor(() => c.querySelector('.trace-header') !== null); + const header = c.querySelector('.trace-header') as HTMLElement; + expect(header.textContent).toContain('4.0s'); + expect(header.textContent).toContain('interrupted'); + }); + it('selects the ?obs= node on load and writes ?obs= with replace on selection', async () => { window.history.replaceState(null, '', '/#/traces/t1?obs=tool1'); daemon({ trace: TRACE }); diff --git a/apps/agent-forge/src/ui/run/RunTree.tsx b/apps/agent-forge/src/ui/run/RunTree.tsx index d00a3dea4..ca1121028 100644 --- a/apps/agent-forge/src/ui/run/RunTree.tsx +++ b/apps/agent-forge/src/ui/run/RunTree.tsx @@ -2,7 +2,7 @@ import type { ReactNode } from 'react'; import { VirtualList } from '../../components/debug/VirtualList'; import { fmtDuration, fmtInt } from '../../observe/format'; -import type { ObservationNode } from '../../observe/tree'; +import { latestKnownEnd, type ObservationNode } from '../../observe/tree'; import type { ObserveObservationRow } from '../../observe/types'; import { StatusPill } from '../StatusPill'; import { @@ -56,8 +56,9 @@ function KindIcon({ kind }: { kind: string }) { ); } -function Duration({ row, now }: { row: ObserveObservationRow; now: number }) { - const ms = elapsedMs(row, now); +function Duration({ node, now }: { node: ObservationNode; now: number }) { + const { row } = node; + const ms = elapsedMs(row, now, !isRunning(row) && row.end === null ? latestKnownEnd(node) : undefined); if (isRunning(row)) return {fmtDuration(ms)} running; return {fmtDuration(ms)}; } @@ -96,7 +97,7 @@ function TreeRow({ node, selected, virtual, now, playgroundLinks, onSelect }: Tr {status !== 'ok' && status !== null && } {row.model && {row.model}} {tokens !== null && {fmtInt(tokens)} tok} - + {playgroundLinks && row.kind === 'generation' && ( diff --git a/apps/agent-forge/src/ui/run/Waterfall.tsx b/apps/agent-forge/src/ui/run/Waterfall.tsx index 67ee98a6c..468de1c19 100644 --- a/apps/agent-forge/src/ui/run/Waterfall.tsx +++ b/apps/agent-forge/src/ui/run/Waterfall.tsx @@ -1,6 +1,6 @@ import { VirtualList } from '../../components/debug/VirtualList'; import { fmtDuration } from '../../observe/format'; -import { traceBounds, type ObservationNode } from '../../observe/tree'; +import { latestKnownEnd, traceBounds, type ObservationNode } from '../../observe/tree'; import type { ObserveObservationRow } from '../../observe/types'; import { VIRTUALISE_ABOVE, VIRTUAL_ROW_HEIGHT, cx, displayName, elapsedMs, isRunning, useFlatNodes } from './rows'; import './run.css'; @@ -62,7 +62,7 @@ interface BarRowProps { function BarRow({ node, minStart, totalMs, now, selected, virtual, onSelect }: BarRowProps) { const { row } = node; - const ms = elapsedMs(row, now); + const ms = elapsedMs(row, now, !isRunning(row) && row.end === null ? latestKnownEnd(node) : undefined); const left = ((row.start - minStart) / totalMs) * 100; const width = Math.max(0.5, (ms / totalMs) * 100); const failed = row.status === 'error'; diff --git a/apps/agent-forge/src/ui/run/__tests__/run.test.tsx b/apps/agent-forge/src/ui/run/__tests__/run.test.tsx index 85630013d..18263a25a 100644 --- a/apps/agent-forge/src/ui/run/__tests__/run.test.tsx +++ b/apps/agent-forge/src/ui/run/__tests__/run.test.tsx @@ -65,6 +65,19 @@ const TRACE: ObserveObservationRow[] = [ const row = (c: HTMLElement, id: string) => c.querySelector(`[data-obs-id="${id}"]`) as HTMLElement; describe('RunTree', () => { + it('treats an interrupted row with no end as terminal: no "running", elapsed stops at the latest known end', () => { + const rows = [ + obs({ obs_id: 'iroot', end: null, status: 'interrupted' }), + obs({ obs_id: 'ichild', parent_obs_id: 'iroot', start: T0 + 10, end: T0 + 250 }), + ]; + const c = render( {}} now={T0 + 60_000} />); + const r = row(c, 'iroot'); + expect(r.querySelector('.run-dur')?.textContent).toBe('250ms'); + expect(r.querySelector('.run-dur-live')).toBeNull(); + expect(r.textContent).not.toContain('running'); + expect(r.textContent).toContain('interrupted'); + }); + it('indents rows by depth in depth-first order', () => { const c = render( {}} now={T0 + 5000} />); const rows = [...c.querySelectorAll('.run-tree-row')]; diff --git a/apps/agent-forge/src/ui/run/rows.ts b/apps/agent-forge/src/ui/run/rows.ts index eaa77206e..592f44437 100644 --- a/apps/agent-forge/src/ui/run/rows.ts +++ b/apps/agent-forge/src/ui/run/rows.ts @@ -30,20 +30,25 @@ export function totalTokens(row: ObserveObservationRow): number | null { return (row.usage_input_tokens ?? 0) + (row.usage_output_tokens ?? 0); } -/** An open observation (`end === null`) is running, whatever its stored status says. */ +/** An open observation (`end === null`) is running unless its stored status is terminal (`interrupted` by the crash sweep, or `error`). */ export function isRunning(row: ObserveObservationRow): boolean { - return row.end === null; + return row.end === null && row.status !== 'interrupted' && row.status !== 'error'; } -/** The status to display: a stored `error` always wins; else `running` for an open observation, else the stored status. */ +/** The status to display: a stored terminal status (`error`, `interrupted`) always wins; else `running` for an open observation, else the stored status. */ export function displayStatus(row: ObserveObservationRow): string | null { - if (row.status === 'error') return 'error'; + if (row.status === 'error' || row.status === 'interrupted') return row.status; return isRunning(row) ? 'running' : row.status; } -/** Elapsed ms: `end - start`, or `now - start` while still running. */ -export function elapsedMs(row: ObserveObservationRow, now: number): number { - return Math.max(0, (row.end ?? now) - row.start); +/** + * Elapsed ms: `end - start`, or `now - start` while still running. An open + * row with a terminal status stops at `fallbackEnd` (the latest known end + * among it and its descendants), else at its start; it never counts to `now`. + */ +export function elapsedMs(row: ObserveObservationRow, now: number, fallbackEnd?: number): number { + const end = row.end ?? (isRunning(row) ? now : (fallbackEnd ?? row.start)); + return Math.max(0, end - row.start); } /** The href that opens a generation in the Playground (it loads the trace, then pre-fills from the observation). */