Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ opt-out.
- Insights runs/day averages over the days an agent was active in the window, so an agent first seen today with 12 runs reports 12/day instead of 12/7.
- An open run's root start is kept when the reporter's buffer trims or a crash flush cuts to its newest 16 MB, so a run whose child spans overflow the buffer is still listed.
- The run inspector shows `error`, not `running`, for a failed step that has no end time.
- Studio trace viewer: an observation the crash sweep marked `interrupted` (or an open `error` one) now shows as terminal instead of "running" forever; its duration and the waterfall stop at the latest known end and the header clock stops ticking.
- The observe daemon refuses cross-site browser writes and DNS rebinding on `/api/*`: a loopback-bound daemon answers only loopback `Host` names (403), writes carrying a non-loopback `Origin` (including `null`) or `Sec-Fetch-Site: cross-site` get 403, and a write with a body must be `Content-Type: application/json` (415) — `curl -d` needs `-H "Content-Type: application/json"`. The SDK reporter, CLI and Studio already comply. See [Observability](docs/observability.md).
- Observe retention (`lousho observe start --retention-days`, `lousho observe prune`) also deletes old traces the sweep marked `interrupted` (a crashed run whose end never arrived); their age counts from when they were interrupted.

Expand Down
6 changes: 6 additions & 0 deletions apps/agent-forge/src/observe/__tests__/tree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ describe('traceBounds', () => {
expect(bounds).toEqual({ minStart: 100, maxEnd: 1000 });
});

it('does not count an open interrupted or error row to now', () => {
expect(
traceBounds([row({ start: 100, end: 400 }), row({ start: 200, end: null, status: 'interrupted' }), row({ start: 300, end: null, status: 'error' })], 1000),
).toEqual({ minStart: 100, maxEnd: 400 });
});

it('is degenerate-safe on empty input', () => {
expect(traceBounds([])).toEqual({ minStart: 0, maxEnd: 1 });
});
Expand Down
17 changes: 15 additions & 2 deletions apps/agent-forge/src/observe/tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,27 @@ export function flattenObservationTree(roots: ObservationNode[]): ObservationNod
return flat;
}

/** `[start, end]` bounds over every observation (an unfinished row counts to `now`). */
/** The latest `end` among a node and its descendants, or `undefined` when none has ended. */
export function latestKnownEnd(node: ObservationNode, seen = new Set<string>()): number | undefined {
if (seen.has(node.row.obs_id)) return undefined;
seen.add(node.row.obs_id);
let latest = node.row.end ?? undefined;
for (const child of node.children) {
const end = latestKnownEnd(child, seen);
if (end !== undefined && (latest === undefined || end > latest)) latest = end;
}
return latest;
}

/** `[start, end]` bounds over every observation (a running row counts to `now`; an open interrupted/error row stops at its start). */
export function traceBounds(rows: ObserveObservationRow[], now = Date.now()): { minStart: number; maxEnd: number } {
if (rows.length === 0) return { minStart: 0, maxEnd: 1 };
let minStart = Infinity;
let maxEnd = -Infinity;
for (const row of rows) {
minStart = Math.min(minStart, row.start);
maxEnd = Math.max(maxEnd, row.end ?? now);
const open = row.status === 'interrupted' || row.status === 'error' ? row.start : now;
maxEnd = Math.max(maxEnd, row.end ?? open);
}
return { minStart, maxEnd: Math.max(maxEnd, minStart + 1) };
}
4 changes: 2 additions & 2 deletions apps/agent-forge/src/screens/queues/QueueReview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import type { ObserveQueue, ObserveQueueItem } from '../../observe/types';
import { useDaemonDown, useNextQueueItem, useQueue, useTraceDetail } from '../../observe/useObserve';
import { EmptyState } from '../../ui/EmptyState';
import { TraceViewer, type TraceViewMode } from '../../ui/run/TraceViewer';
import { displayName } from '../../ui/run/rows';
import { displayName, isRunning } from '../../ui/run/rows';
import { useNow } from '../../ui/run/useNow';
import { fmtInt } from '../../observe/format';
import { RubricForm } from './RubricForm';
Expand All @@ -38,7 +38,7 @@ function ReviewItem({ queue, item, onDone }: { queue: ObserveQueue; item: Observ
const [mode, setMode] = useState<TraceViewMode>('tree');
const observations = trace.data?.observations;
const root = useMemo(() => (observations ? nestObservations(observations)[0]?.row : undefined), [observations]);
const now = useNow(root?.end === null);
const now = useNow(root !== undefined && isRunning(root));

let body;
if (down) body = <ObserveDaemonDown />;
Expand Down
9 changes: 8 additions & 1 deletion apps/agent-forge/src/screens/traces/TraceHeader.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ import { displayName, displayStatus, elapsedMs } from '../../ui/run/rows';
import { AddToDataset } from './AddToDataset';
import { AddToQueue } from './AddToQueue';

/** The latest `end` over every observation, or `undefined` when none has ended. */
function latestEnd(rows: ObserveObservationRow[]): number | undefined {
let latest: number | undefined;
for (const row of rows) if (row.end !== null && (latest === undefined || row.end > latest)) latest = row.end;
return latest;
}

/** Trace-wide totals, summed over every observation the way the daemon's trace list does. */
function traceTotals(rows: ObserveObservationRow[]): { tokens: number | null; cost: number | null } {
let tokens: number | null = null;
Expand Down Expand Up @@ -88,7 +95,7 @@ export function TraceHeader({
)}
<Fact label="Model">{model ? <code>{model}</code> : '-'}</Fact>
<Fact label="Start">{fmtDateTime(root.start)}</Fact>
<Fact label="Duration">{fmtDuration(elapsedMs(root, now))}</Fact>
<Fact label="Duration">{fmtDuration(elapsedMs(root, now, latestEnd(observations)))}</Fact>
<Fact label="Tokens">{fmtInt(tokens)}</Fact>
<Fact label="Cost">{fmtCost(cost)}</Fact>
{root.session_id && (
Expand Down
3 changes: 2 additions & 1 deletion apps/agent-forge/src/screens/traces/TraceScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { useQueryState } from '../../router/useRoute';
import { EmptyState } from '../../ui/EmptyState';
import { tracesListPath } from './tracesUrl';
import { TraceViewer, type TraceViewMode } from '../../ui/run/TraceViewer';
import { isRunning } from '../../ui/run/rows';
import { useNow } from '../../ui/run/useNow';
import { TraceFeedback } from './TraceFeedback';
import { TraceHeader } from './TraceHeader';
Expand Down Expand Up @@ -49,7 +50,7 @@ function useTrace(traceId: string, pollMs: number) {

const observations = detail.data?.observations;
const root = useMemo(() => (observations ? rootOf(observations) : undefined), [observations]);
const now = useNow(root?.end === null);
const now = useNow(root !== undefined && isRunning(root));
const agentName = agents.data?.find((a) => a.agent_id === root?.agent_id)?.name ?? undefined;
return { detail, down, root, now, agentName };
}
Expand Down
18 changes: 18 additions & 0 deletions apps/agent-forge/src/screens/traces/__tests__/TraceScreen.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,24 @@ describe('TraceScreen', () => {
expect(header.querySelector('button[aria-label="Copy link"]')).not.toBeNull();
});

it('header duration for an interrupted root with no end stops at the latest child end', async () => {
daemon({
trace: {
trace_id: 't1',
observations: [
obs({ obs_id: 'root', name: 'crashed run', start: T0, end: null, status: 'interrupted' }),
obs({ obs_id: 'c1', parent_obs_id: 'root', start: T0 + 100, end: T0 + 4000 }),
],
scores: [],
},
});
const c = mount();
await waitFor(() => c.querySelector('.trace-header') !== null);
const header = c.querySelector('.trace-header') as HTMLElement;
expect(header.textContent).toContain('4.0s');
expect(header.textContent).toContain('interrupted');
});

it('selects the ?obs= node on load and writes ?obs= with replace on selection', async () => {
window.history.replaceState(null, '', '/#/traces/t1?obs=tool1');
daemon({ trace: TRACE });
Expand Down
9 changes: 5 additions & 4 deletions apps/agent-forge/src/ui/run/RunTree.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type { ReactNode } from 'react';

import { VirtualList } from '../../components/debug/VirtualList';
import { fmtDuration, fmtInt } from '../../observe/format';
import type { ObservationNode } from '../../observe/tree';
import { latestKnownEnd, type ObservationNode } from '../../observe/tree';
import type { ObserveObservationRow } from '../../observe/types';
import { StatusPill } from '../StatusPill';
import {
Expand Down Expand Up @@ -56,8 +56,9 @@ function KindIcon({ kind }: { kind: string }) {
);
}

function Duration({ row, now }: { row: ObserveObservationRow; now: number }) {
const ms = elapsedMs(row, now);
function Duration({ node, now }: { node: ObservationNode; now: number }) {
const { row } = node;
const ms = elapsedMs(row, now, !isRunning(row) && row.end === null ? latestKnownEnd(node) : undefined);
if (isRunning(row)) return <span className="run-dur run-dur-live">{fmtDuration(ms)} running</span>;
return <span className="run-dur">{fmtDuration(ms)}</span>;
}
Expand Down Expand Up @@ -96,7 +97,7 @@ function TreeRow({ node, selected, virtual, now, playgroundLinks, onSelect }: Tr
{status !== 'ok' && status !== null && <StatusPill status={status} />}
{row.model && <span className="run-model-chip">{row.model}</span>}
{tokens !== null && <span className="run-tokens">{fmtInt(tokens)} tok</span>}
<Duration row={row} now={now} />
<Duration node={node} now={now} />
</button>
{playgroundLinks && row.kind === 'generation' && (
<a className="run-playground-link" href={playgroundHref(row.trace_id, row.obs_id)} title={`Open ${name} in Playground`}>
Expand Down
4 changes: 2 additions & 2 deletions apps/agent-forge/src/ui/run/Waterfall.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { VirtualList } from '../../components/debug/VirtualList';
import { fmtDuration } from '../../observe/format';
import { traceBounds, type ObservationNode } from '../../observe/tree';
import { latestKnownEnd, traceBounds, type ObservationNode } from '../../observe/tree';
import type { ObserveObservationRow } from '../../observe/types';
import { VIRTUALISE_ABOVE, VIRTUAL_ROW_HEIGHT, cx, displayName, elapsedMs, isRunning, useFlatNodes } from './rows';
import './run.css';
Expand Down Expand Up @@ -62,7 +62,7 @@ interface BarRowProps {

function BarRow({ node, minStart, totalMs, now, selected, virtual, onSelect }: BarRowProps) {
const { row } = node;
const ms = elapsedMs(row, now);
const ms = elapsedMs(row, now, !isRunning(row) && row.end === null ? latestKnownEnd(node) : undefined);
const left = ((row.start - minStart) / totalMs) * 100;
const width = Math.max(0.5, (ms / totalMs) * 100);
const failed = row.status === 'error';
Expand Down
13 changes: 13 additions & 0 deletions apps/agent-forge/src/ui/run/__tests__/run.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,19 @@ const TRACE: ObserveObservationRow[] = [
const row = (c: HTMLElement, id: string) => c.querySelector(`[data-obs-id="${id}"]`) as HTMLElement;

describe('RunTree', () => {
it('treats an interrupted row with no end as terminal: no "running", elapsed stops at the latest known end', () => {
const rows = [
obs({ obs_id: 'iroot', end: null, status: 'interrupted' }),
obs({ obs_id: 'ichild', parent_obs_id: 'iroot', start: T0 + 10, end: T0 + 250 }),
];
const c = render(<RunTree observations={rows} onSelect={() => {}} now={T0 + 60_000} />);
const r = row(c, 'iroot');
expect(r.querySelector('.run-dur')?.textContent).toBe('250ms');
expect(r.querySelector('.run-dur-live')).toBeNull();
expect(r.textContent).not.toContain('running');
expect(r.textContent).toContain('interrupted');
});

it('indents rows by depth in depth-first order', () => {
const c = render(<RunTree observations={TRACE} onSelect={() => {}} now={T0 + 5000} />);
const rows = [...c.querySelectorAll<HTMLElement>('.run-tree-row')];
Expand Down
19 changes: 12 additions & 7 deletions apps/agent-forge/src/ui/run/rows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,25 @@ export function totalTokens(row: ObserveObservationRow): number | null {
return (row.usage_input_tokens ?? 0) + (row.usage_output_tokens ?? 0);
}

/** An open observation (`end === null`) is running, whatever its stored status says. */
/** An open observation (`end === null`) is running unless its stored status is terminal (`interrupted` by the crash sweep, or `error`). */
export function isRunning(row: ObserveObservationRow): boolean {
return row.end === null;
return row.end === null && row.status !== 'interrupted' && row.status !== 'error';
}

/** The status to display: a stored `error` always wins; else `running` for an open observation, else the stored status. */
/** The status to display: a stored terminal status (`error`, `interrupted`) always wins; else `running` for an open observation, else the stored status. */
export function displayStatus(row: ObserveObservationRow): string | null {
if (row.status === 'error') return 'error';
if (row.status === 'error' || row.status === 'interrupted') return row.status;
return isRunning(row) ? 'running' : row.status;
}

/** Elapsed ms: `end - start`, or `now - start` while still running. */
export function elapsedMs(row: ObserveObservationRow, now: number): number {
return Math.max(0, (row.end ?? now) - row.start);
/**
* Elapsed ms: `end - start`, or `now - start` while still running. An open
* row with a terminal status stops at `fallbackEnd` (the latest known end
* among it and its descendants), else at its start; it never counts to `now`.
*/
export function elapsedMs(row: ObserveObservationRow, now: number, fallbackEnd?: number): number {
const end = row.end ?? (isRunning(row) ? now : (fallbackEnd ?? row.start));
return Math.max(0, end - row.start);
}

/** The href that opens a generation in the Playground (it loads the trace, then pre-fills from the observation). */
Expand Down
Loading