diff --git a/.fallowrc.json b/.fallowrc.json index 593760c2..439f702d 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -1,7 +1,7 @@ { "$schema": "./node_modules/fallow/schema.json", "entry": [ - "src/cli/{dev,chat,build,studio,mcp,doctor,init,eval}.ts", + "src/cli/{dev,chat,build,studio,mcp,doctor,init,eval,observe,prompt}.ts", "**/*.test-d.ts", "**/*.{test,spec}.{ts,tsx}", "**/*.eval.ts", @@ -20,6 +20,7 @@ "src/deploy/shims/google.worker.ts", "src/deploy/shims/mcp.worker.ts", "src/deploy/shims/pi.worker.ts", + "src/deploy/shims/isolatedVm.worker.ts", "src/agentDir/__fixtures__/**/*.{ts,js}", "examples/agent-dir/tools/*.ts", "registry/**/*.ts" @@ -94,7 +95,8 @@ "ignore": [ "**/*.test-d.ts", "registry/**", - "examples/**" + "examples/**", + "apps/agent-forge/src/observe/types.ts" ] }, "health": { @@ -183,6 +185,59 @@ ], "maxCrap": 80, "reason": "Low-complexity helpers whose per-function CRAP is inflated because fallow's coverage matching scores parts of them 0%-covered despite tests exercising them through higher-level calls (eval reporting, dev event formatting, the warn-once path). Ceiling exempts only the coverage term; cyclomatic/cognitive/unit-size still gate." + }, + { + "files": [ + "src/observe/reporter.ts" + ], + "maxCyclomatic": 35, + "maxCognitive": 30, + "maxCrap": 35, + "reason": "observationOf is a flat attribute-mapping ladder (~15 optional gen_ai.* fields, one conditional each) and observeExporter a reporter factory; both are covered by reporter.test.ts (at 100% the CRAP floor equals cyclomatic). Splitting the ladder would scatter one schema across helpers without removing a decision. Unit-size still gates." + }, + { + "files": [ + "src/observe/sweep.ts" + ], + "maxCyclomatic": 25, + "maxCognitive": 32, + "maxCrap": 140, + "reason": "sweepAgents' healthy/stale/down/interrupt status ladder is exercised by sweep.test.ts, but fallow scores its nested row callbacks at the uncovered estimate. Ceilings exempt the status-derivation density and the coverage term; unit-size still gates." + }, + { + "files": [ + "src/observe/store.ts" + ], + "maxCognitive": 20, + "maxCrap": 220, + "reason": "The row mappers in listAgents/insightsAgents and the upsert/rollup SQL in writeObservation/observationRow run under store.test.ts and the route tests, but fallow's per-function matching scores nested arrows 0%-covered. Ceilings exempt the coverage term and the dense SQL assembly; cyclomatic and unit-size still gate." + }, + { + "files": [ + "src/observe/identity.ts" + ], + "maxCognitive": 28, + "maxCrap": 40, + "reason": "resolveIdentity's hostname/pid/git/persisted-uuid ladder is covered through the reporter tests (~57% - the fs/git probe failure rungs vary by platform); each rung is one guarded probe. Ceilings exempt the probe density and the coverage term; cyclomatic and unit-size still gate." + }, + { + "files": [ + "src/observe/emit.ts", + "src/observe/daemon.ts", + "src/observe/routes/hub.ts", + "src/observe/routes/prompts.ts" + ], + "maxCrap": 100, + "reason": "Covered by the emit/daemon/hub/prompts test files; the flagged handlers and nested callbacks fall below fallow's function-level coverage matching so it scores them at the uncovered estimate. Ceiling exempts only the coverage term; cyclomatic/cognitive/unit-size still gate." + }, + { + "files": [ + "src/cli/observe.ts", + "src/cli/prompt.ts" + ], + "maxCognitive": 28, + "maxCrap": 60, + "reason": "CLI glue: observe.test.ts covers status/stop/token/rebuild/collision and prompt.test.ts covers push/list/get/env-port, but runStart's post-listen announce lines can't run in-process (a started daemon has no closeable handle and keeps the temp db locked). Ceiling exempts the arg-handling density and the coverage term; cyclomatic/unit-size still gate." } ] } diff --git a/.gitignore b/.gitignore index cabe8c56..154c7156 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,6 @@ audit/ tmp-*/ npmci-*.log snip.txt + +# Smoke-test artifacts +.smoke/shots/ diff --git a/.smoke/agent.ts b/.smoke/agent.ts new file mode 100644 index 00000000..edec8058 --- /dev/null +++ b/.smoke/agent.ts @@ -0,0 +1,45 @@ +/** + * Live smoke test for the observability platform: prompt registry fetch + + * prompt_use, tool-call observation, hub publish/query, all reported to the + * daemon via LOUSHO_OBSERVE=1. Run: npx tsx .smoke/agent.ts + */ +import { createAgent, defineTool, hub } from '../src/index'; +import { z } from 'zod'; + +const getTime = defineTool({ + name: 'get_time', + description: 'Get the current date and time in ISO format.', + input: z.object({}), + execute: async () => new Date().toISOString(), +}); + +async function main() { + const h = hub(); + if (!h) throw new Error('hub() is undefined - LOUSHO_OBSERVE not set?'); + await h.publish({ namespace: 'smoke', key: 'last-run', value: new Date().toISOString(), scope: 'fleet' }); + const entries = await h.query({ namespace: 'smoke', scope: 'fleet' }); + console.log('[hub] fleet entries in smoke namespace:', entries.length); + + const agent = createAgent({ + name: 'smoke-agent', + model: 'openrouter/nvidia/nemotron-3-ultra-550b-a55b:free', + prompt: 'observe-test@latest', + tools: { get_time: getTime }, + }); + + const first = await agent.send('Hey! What time is it right now?'); + console.log('[agent 1]', first.text); + const second = await agent.send('And what day of the week is that?'); + console.log('[agent 2]', second.text); + + // Give the reporter's interval flush a moment, then close for the exit flush. + const reporter = (agent as unknown as { __observeReporter?: { close(): Promise } }).__observeReporter; + await reporter?.close(); + await new Promise((r) => setTimeout(r, 3000)); + console.log('[smoke] done'); +} + +main().catch((e) => { + console.error('[smoke] failed:', e); + process.exit(1); +}); diff --git a/.smoke/ui-check.mts b/.smoke/ui-check.mts new file mode 100644 index 00000000..8861de25 --- /dev/null +++ b/.smoke/ui-check.mts @@ -0,0 +1,71 @@ +/** + * Drive the built Studio Observe UI end to end against the live daemon. + * Run from apps/agent-forge: npx tsx .smoke-ui.mts + */ +import { chromium } from 'playwright'; + +const BASE = 'http://127.0.0.1:4752/?token=vL_69opXZD_HsceHt608DJaQDxWrDzcy'; +const shots = 'E:/agent-sdk/.smoke/shots'; +const results: string[] = []; +async function check(name: string, fn: () => Promise) { + try { + results.push(`${(await fn()) ? 'PASS' : 'FAIL'} ${name}`); + } catch (e) { + results.push(`FAIL ${name} (${e instanceof Error ? e.message : e})`); + } +} +const text = (page: import('playwright').Page, sel: string) => page.locator(sel).first().textContent({ timeout: 5000 }).catch(() => null); + +const browser = await chromium.launch(); +const page = await browser.newPage({ viewport: { width: 1400, height: 900 } }); +const errors: string[] = []; +page.on('pageerror', (e) => errors.push(e.message)); +page.on('console', (m) => { if (m.type() === 'error') errors.push(m.text()); }); + +await page.goto(BASE, { waitUntil: 'networkidle' }); +await check('studio loads', async () => (await page.title()) !== ''); + +// Flip to the Observe surface. +await page.getByRole('button', { name: 'Observe' }).first().click(); +await page.waitForSelector('section.observe', { timeout: 10_000 }); +await check('observe tabs render', async () => (await page.getByRole('tab').count()) === 4); + +// --- Traces tab --- +await page.waitForTimeout(1500); +await check('traces list has smoke-agent rows', async () => (await page.getByText('invoke_agent smoke-agent').count()) >= 2); +await page.screenshot({ path: `${shots}/01-traces.png`, fullPage: true }); + +// Open the first trace -> waterfall detail. +await page.getByText('invoke_agent smoke-agent').first().click(); +await page.waitForTimeout(1200); +await check('trace detail shows generation obs', async () => (await text(page, 'body'))?.includes('chat nvidia') ?? false); +await check('trace detail shows tool call', async () => (await text(page, 'body'))?.includes('get_time') ?? false); +await page.screenshot({ path: `${shots}/02-trace-detail.png`, fullPage: true }); +await page.getByRole('button', { name: /back|traces/i }).first().click().catch(() => page.keyboard.press('Escape')); +await page.waitForTimeout(600); + +// --- Insights tab --- +await page.getByRole('tab', { name: 'Insights' }).click(); +await page.waitForTimeout(1200); +await check('insights shows agent card', async () => (await text(page, 'body'))?.includes('smoke-agent') ?? false); +await check('insights shows latency figure', async () => /p50|p95|3825|4135/i.test((await text(page, 'body')) ?? '')); +await page.screenshot({ path: `${shots}/03-insights.png`, fullPage: true }); + +// --- Monitoring tab --- +await page.getByRole('tab', { name: 'Monitoring' }).click(); +await page.waitForTimeout(1200); +await check('fleet table has smoke-agent', async () => (await text(page, 'body'))?.includes('smoke-agent') ?? false); +await check('fleet table has status', async () => /stopped|healthy|stale|down/i.test((await text(page, 'body')) ?? '')); +await page.screenshot({ path: `${shots}/04-monitoring.png`, fullPage: true }); + +// --- Scores tab --- +await page.getByRole('tab', { name: 'Scores' }).click(); +await page.waitForTimeout(1200); +await check('scores shows eval row', async () => (await text(page, 'body'))?.includes('smoke: eval') ?? false); +await page.screenshot({ path: `${shots}/05-scores.png`, fullPage: true }); + +await check('no page errors', async () => errors.length === 0); + +console.log(results.join('\n')); +if (errors.length) console.log('\npage errors:\n' + errors.slice(0, 5).join('\n')); +await browser.close(); diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee43627..33e403ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ What is on `main` and not yet on npm. Entries marked **Behavior change** can break working code; [Upgrading](docs/upgrading.md) lists each one with its opt-out. +### Added + +- The `lousho observe` daemon: a local collector (loopback, `http://127.0.0.1:7777`, SQLite at `~/.lousho/observe.db`) that gathers every agent's traces, heartbeats and eval scores when `LOUSHO_OBSERVE` is set — no per-agent code changes. `lousho observe start|status|stop|rebuild-rollups|token create`; a second `start` on a busy port attaches when it is already the observe daemon. See [Observability](docs/observability.md#the-observe-daemon). +- Agent Forge gains Observe tabs — Traces, Insights, Monitoring, Scores — reading the daemon through a token-gated `/observe/**` proxy on the studio server. +- Prompt registry: `createAgent({ prompt: 'name@version' })` or `LOUSHO_PROMPT` resolves versioned instructions from the daemon at build (embedded `instructions` are the fallback); every run records the version used. `lousho prompt push|list|get` manages it. +- Context hub: `hub()` / `hubTools()` publish and query namespaced shared memory scoped `private` (the writing agent) or `fleet` (every agent on the daemon). +- Fleet monitoring: heartbeat-derived agent status, `agent.deregister` on graceful exit, a crashed agent's open sessions and unfinished observations flip to `interrupted`, and webhook alerts (`--webhook`, `alerts.json`) on down transitions and error-rate thresholds. +- Eval scores report to the daemon: every `AssertionResult` lands as a score row on the exercised run's trace (`source: eval` or `judge`); `lousho eval --observe` forces emission. +- `sandbox({ driver: 'isolate' })` runs tool code in an `isolated-vm` V8 isolate (optional peer `isolated-vm`); `defineTool({ sandbox })` opts a tool in, timeouts and OOM map to `sandbox.error` on the tool span. `'docker'` and `'e2b'` driver names are reserved for later drivers. + +## [1.0.0-alpha.29] - 2026-10-21 + ### Changed - **Behavior change:** the built node-server and Docker image refuse to start on a non-loopback host when no auth is configured (no `LOUSHO_API_TOKEN`, build token or `auth.ts`), with `LOUSHO_SERVER_UNPROTECTED`. Set a token, add `auth.ts`, bind to `127.0.0.1`, or set `LOUSHO_ALLOW_PUBLIC=1` to serve an open agent on purpose. `lousho build --target=docker` prints the run command with `-e LOUSHO_API_TOKEN=`. diff --git a/api/executor.api.md b/api/executor.api.md index 7aada9bd..b685d8b0 100644 --- a/api/executor.api.md +++ b/api/executor.api.md @@ -2031,6 +2031,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -2625,6 +2642,7 @@ interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -2736,6 +2754,13 @@ interface ToolResumeEvent extends AgentEventBase<'tool.resume'> { toolName: string; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public interface ToolSearchOptions { contextWindow?: number; @@ -2764,9 +2789,7 @@ interface ToolStartEvent extends AgentEventBase<'tool.start'> { // @public interface TraceExporter { - // (undocumented) onSpanEnd(span: Span): void; - // (undocumented) onSpanStart(span: Span): void; } diff --git a/api/flows.api.md b/api/flows.api.md index 883ab8ee..7817bddd 100644 --- a/api/flows.api.md +++ b/api/flows.api.md @@ -1162,6 +1162,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -1407,6 +1424,7 @@ interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -1465,14 +1483,19 @@ class ToolRegistry { unregister(name: string): boolean; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public type ToolsOption = ReadonlyArray | Record; // @public interface TraceExporter { - // (undocumented) onSpanEnd(span: Span): void; - // (undocumented) onSpanStart(span: Span): void; } diff --git a/api/index.api.md b/api/index.api.md index 906b1da2..2dac032a 100644 --- a/api/index.api.md +++ b/api/index.api.md @@ -701,6 +701,7 @@ export interface AssertionResult { passed: boolean; score: number; skipped?: boolean; + source?: 'eval' | 'judge'; threshold?: number; } @@ -1454,7 +1455,7 @@ export function createAgentConfigOf(agent: SimpleAgent): CreateAgentConfig | und // @public export type CreateAgentInstructions = { instructions?: PerRun; - prompt?: undefined; + prompt?: PromptRefString; } | { instructions?: undefined; prompt?: PerRun; @@ -1700,6 +1701,7 @@ export interface DefineToolOptions { name: string; needsApproval?: boolean | ((args: InferSchemaOutput, ctx: ApprovalCheckContext) => ApprovalOutcome | Promise); requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: InferSchemaOutput, sandbox: SandboxAdapter) => Promise; timeoutMs?: number; } @@ -1971,6 +1973,7 @@ export const ERROR_CODES: { readonly LOUSHO_RUN_ALREADY_ITERATED: "Iterate an AgentRun once; call stream() again for a new run."; readonly LOUSHO_SANDBOX_PATH_DENIED: "Write under the sandbox's root (SubprocessSandbox({ root })), or pass the root that should contain the path."; readonly LOUSHO_SANDBOX_EGRESS_UNSUPPORTED: "Run on Docker Engine 25.0.5+ for Linux on this host (not Docker Desktop, rootless or a remote daemon), or use network: 'none'."; + readonly LOUSHO_SANDBOX_FAILED: "The error's `kind` says which boundary failed ('timeout', 'oom' or 'driver'); raise the limit in sandbox()/defineTool({ sandbox }), or check the driver."; readonly LOUSHO_AGENT_EXECUTION_FAILED: "Look at the `cause` for the underlying failure."; readonly LOUSHO_FLOW_EXECUTION_FAILED: "Look at the failing `step` and the `cause`."; readonly LOUSHO_VALIDATION_FAILED: "Fix the fields listed in `errors`."; @@ -2078,6 +2081,7 @@ export interface EvalResult { export interface EvalTarget { send(input: AgentInput, options?: { onEvent?: (event: AgentEvent) => void; + parentSpanId?: string; }): Promise; } @@ -3114,6 +3118,68 @@ export interface HttpToolOptions { validateSSL?: boolean; } +// @public +export interface Hub { + readonly agentId: string; + publish(input: HubPublishInput): Promise; + query(filter?: HubQuery): Promise; +} + +// @public +export function hub(options?: HubOptions): Hub | undefined; + +// @public +export const HUB_SCOPES: readonly ["private", "fleet"]; + +// @public +export interface HubEntry { + agent_id: string | null; + created_at: number; + entry_id: string; + expires_at: number | null; + key: string; + namespace: string; + scope: string; + ttl: number | null; + value: string | null; +} + +// @public +export interface HubOptions { + agentId?: string; + agentName?: string; + cwd?: string; + env?: NodeJS.ProcessEnv; + fetchFn?: typeof fetch; + project?: string; + timeoutMs?: number; + token?: string; + url?: string; +} + +// @public +export interface HubPublishInput { + key: string; + namespace: string; + scope?: HubScope; + ttl?: number; + value: unknown; +} + +// @public +export interface HubQuery { + key?: string; + limit?: number; + namespace?: string; + scope?: HubScope; +} + +// @public +export type HubScope = (typeof HUB_SCOPES)[number]; + +// @public +export function hubTools(options?: HubOptions): DefinedTool[]; + // @public export interface ImageContentPart { // (undocumented) @@ -3252,6 +3318,9 @@ export function isHandoff(value: unknown): value is Handoff; // @public export function isHostedTool(value: unknown): value is HostedTool; +// @public +export function isJudgeScorer(fn: unknown): fn is (result: ExecutionResult) => Promise; + // @public export function isModelActionableProviderErrorCategory(category: CompactedProviderErrorCategory): boolean; @@ -4870,6 +4939,9 @@ export function promptInjectionGuardrail(options?: { runInParallel?: boolean; }): IoGuardrail; +// @public +export type PromptRefString = `${string}@${number}` | `${string}@latest`; + // @public export interface PromptScheduleInput extends ScheduleBase { // (undocumented) @@ -5439,6 +5511,15 @@ interface SafeParser { }; } +// @public +export interface Sandbox { + readonly driver: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +export function sandbox(options: SandboxOptions): Sandbox; + // @public export interface SandboxAdapter { readonly name: string; @@ -5446,6 +5527,50 @@ export interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +export const SandboxAttr: { + readonly DRIVER: "sandbox.driver"; + readonly ERROR: "sandbox.error"; +}; + +// @public +export interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +export interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +export type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +export function sandboxDriverName(driver: SandboxDriverRef | undefined): string; + +// @public +export type SandboxDriverRef = SandboxDriverName | SandboxDriver; + +// @public +export class SandboxError extends SDKError { + constructor(message: string, + kind: SandboxErrorKind, + driver: string, options?: { + cause?: unknown; + }); + readonly driver: string; + readonly kind: SandboxErrorKind; + readonly toolErrorKind: "sandbox"; +} + +// @public +export type SandboxErrorKind = 'timeout' | 'oom' | 'driver'; + +// @public +export function sandboxErrorKind(error: unknown, depth?: number): SandboxErrorKind | undefined; + // @public export interface SandboxLimits { cpus?: number; @@ -5458,6 +5583,13 @@ export type SandboxNetwork = 'none' | 'default' | { allow: readonly string[]; }; +// @public +export interface SandboxOptions { + driver: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public export interface SandboxResult { // (undocumented) @@ -6750,6 +6882,7 @@ export interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -6810,6 +6943,7 @@ export interface ToolErrorResult { error: string; kind: ToolErrorKind; message: string; + sandboxError?: string; toolCallId?: string; // (undocumented) toolName: string; @@ -6937,6 +7071,13 @@ interface ToolResumeEvent extends AgentEventBase<'tool.resume'> { toolName: string; } +// @public +export interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public export interface ToolScopeOptions { subagent?: string; @@ -6985,9 +7126,7 @@ export function toUIMessageStreamResponse(run: AsyncIterable, init?: // @public export interface TraceExporter { - // (undocumented) onSpanEnd(span: Span): void; - // (undocumented) onSpanStart(span: Span): void; } diff --git a/api/integrations.api.md b/api/integrations.api.md index e149b45b..48ec1a17 100644 --- a/api/integrations.api.md +++ b/api/integrations.api.md @@ -435,6 +435,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -565,6 +582,7 @@ interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -613,6 +631,13 @@ class ToolRegistry { unregister(name: string): boolean; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public type ToolsOption = ReadonlyArray | Record; diff --git a/api/mcp.api.md b/api/mcp.api.md index 7a29f6fb..1a44a29f 100644 --- a/api/mcp.api.md +++ b/api/mcp.api.md @@ -610,6 +610,7 @@ const ERROR_CODES: { readonly LOUSHO_RUN_ALREADY_ITERATED: "Iterate an AgentRun once; call stream() again for a new run."; readonly LOUSHO_SANDBOX_PATH_DENIED: "Write under the sandbox's root (SubprocessSandbox({ root })), or pass the root that should contain the path."; readonly LOUSHO_SANDBOX_EGRESS_UNSUPPORTED: "Run on Docker Engine 25.0.5+ for Linux on this host (not Docker Desktop, rootless or a remote daemon), or use network: 'none'."; + readonly LOUSHO_SANDBOX_FAILED: "The error's `kind` says which boundary failed ('timeout', 'oom' or 'driver'); raise the limit in sandbox()/defineTool({ sandbox }), or check the driver."; readonly LOUSHO_AGENT_EXECUTION_FAILED: "Look at the `cause` for the underlying failure."; readonly LOUSHO_FLOW_EXECUTION_FAILED: "Look at the failing `step` and the `cause`."; readonly LOUSHO_VALIDATION_FAILED: "Fix the fields listed in `errors`."; @@ -1680,6 +1681,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -2137,6 +2155,7 @@ interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -2218,6 +2237,13 @@ interface ToolResumeEvent extends AgentEventBase<'tool.resume'> { toolName: string; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public interface ToolStartEvent extends AgentEventBase<'tool.start'> { args: Record; diff --git a/api/otel.api.md b/api/otel.api.md index 51e6433f..abfab6d0 100644 --- a/api/otel.api.md +++ b/api/otel.api.md @@ -51,9 +51,7 @@ interface SpanStatus { // @public interface TraceExporter { - // (undocumented) onSpanEnd(span: Span): void; - // (undocumented) onSpanStart(span: Span): void; } diff --git a/api/testing.api.md b/api/testing.api.md index cbd95e2b..7b2d5b54 100644 --- a/api/testing.api.md +++ b/api/testing.api.md @@ -132,6 +132,7 @@ const ERROR_CODES: { readonly LOUSHO_RUN_ALREADY_ITERATED: "Iterate an AgentRun once; call stream() again for a new run."; readonly LOUSHO_SANDBOX_PATH_DENIED: "Write under the sandbox's root (SubprocessSandbox({ root })), or pass the root that should contain the path."; readonly LOUSHO_SANDBOX_EGRESS_UNSUPPORTED: "Run on Docker Engine 25.0.5+ for Linux on this host (not Docker Desktop, rootless or a remote daemon), or use network: 'none'."; + readonly LOUSHO_SANDBOX_FAILED: "The error's `kind` says which boundary failed ('timeout', 'oom' or 'driver'); raise the limit in sandbox()/defineTool({ sandbox }), or check the driver."; readonly LOUSHO_AGENT_EXECUTION_FAILED: "Look at the `cause` for the underlying failure."; readonly LOUSHO_FLOW_EXECUTION_FAILED: "Look at the failing `step` and the `cause`."; readonly LOUSHO_VALIDATION_FAILED: "Fix the fields listed in `errors`."; diff --git a/api/tools.api.md b/api/tools.api.md index f97a8345..305b4d72 100644 --- a/api/tools.api.md +++ b/api/tools.api.md @@ -595,6 +595,7 @@ export interface DefineToolOptions { name: string; needsApproval?: boolean | ((args: InferSchemaOutput, ctx: ApprovalCheckContext) => ApprovalOutcome | Promise); requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: InferSchemaOutput, sandbox: SandboxAdapter) => Promise; timeoutMs?: number; } @@ -705,6 +706,7 @@ const ERROR_CODES: { readonly LOUSHO_RUN_ALREADY_ITERATED: "Iterate an AgentRun once; call stream() again for a new run."; readonly LOUSHO_SANDBOX_PATH_DENIED: "Write under the sandbox's root (SubprocessSandbox({ root })), or pass the root that should contain the path."; readonly LOUSHO_SANDBOX_EGRESS_UNSUPPORTED: "Run on Docker Engine 25.0.5+ for Linux on this host (not Docker Desktop, rootless or a remote daemon), or use network: 'none'."; + readonly LOUSHO_SANDBOX_FAILED: "The error's `kind` says which boundary failed ('timeout', 'oom' or 'driver'); raise the limit in sandbox()/defineTool({ sandbox }), or check the driver."; readonly LOUSHO_AGENT_EXECUTION_FAILED: "Look at the `cause` for the underlying failure."; readonly LOUSHO_FLOW_EXECUTION_FAILED: "Look at the failing `step` and the `cause`."; readonly LOUSHO_VALIDATION_FAILED: "Fix the fields listed in `errors`."; @@ -2097,6 +2099,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -2669,6 +2688,7 @@ interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -2776,6 +2796,13 @@ interface ToolResumeEvent extends AgentEventBase<'tool.resume'> { toolName: string; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public export type ToolsOption = ReadonlyArray | Record; diff --git a/api/traces.api.md b/api/traces.api.md index 006f0852..333f433a 100644 --- a/api/traces.api.md +++ b/api/traces.api.md @@ -7,6 +7,9 @@ // @public export const DEFAULT_TRACE_DIR = ".lousho/traces"; +// @public +export function fanOutExporters(exporters: readonly (TraceExporter | undefined | null)[]): TraceExporter | undefined; + // @public export function fileTraceExporter(options?: FileTraceExporterOptions): TraceExporter; @@ -24,6 +27,61 @@ export interface ListTracesOptions { limit?: number; } +// @public +export function observeExporter(options?: ObserveExporterOptions): ObserveReporter; + +// @public +export function observeExporterFromEnv(options?: { + agentName?: string; + promptUse?: PromptUseProvider; + env?: NodeJS.ProcessEnv; +}): ObserveReporter | undefined; + +// @public +export interface ObserveExporterOptions { + agentName?: string; + cwd?: string; + fetchFn?: typeof fetch; + flushIntervalMs?: number; + flushSize?: number; + heartbeatIntervalMs?: number; + maxBuffered?: number; + now?: () => number; + project?: string; + promptUse?: PromptUseProvider; + registerExitHooks?: boolean; + requestTimeoutMs?: number; + sessionId?: string; + token?: string; + url?: string; + version?: string; +} + +// @public (undocumented) +export interface ObserveReporter extends TraceExporter { + readonly agentId: string; + close(): Promise; + readonly dropped: number; + promptUse?: PromptUseProvider; + readonly sessionId: string; +} + +// @public +interface PromptUseData { + // (undocumented) + obs_id: string; + // (undocumented) + prompt_id: string; + // (undocumented) + version?: number; +} + +// @public +type PromptUseProvider = () => PromptUseRef | undefined; + +// @public +type PromptUseRef = Pick; + // @public export function readTrace(traceId: string, options?: { dir?: string; @@ -59,9 +117,7 @@ export interface SpanStatus { // @public export interface TraceExporter { - // (undocumented) onSpanEnd(span: Span): void; - // (undocumented) onSpanStart(span: Span): void; } @@ -107,6 +163,11 @@ export interface TraceSummary { traceId: string; } +// @public +export function withObserveExporter(explicit: TraceExporter | undefined, agentName: string | undefined, options?: { + promptUse?: PromptUseProvider; +}): TraceExporter | undefined; + // (No @packageDocumentation comment for this package) ``` diff --git a/api/types.api.md b/api/types.api.md index 68bb0652..9ccd073a 100644 --- a/api/types.api.md +++ b/api/types.api.md @@ -491,6 +491,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -635,6 +652,7 @@ export interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -701,6 +719,13 @@ export interface ToolParameters { type: string; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public (undocumented) interface UIComponentNode { // (undocumented) diff --git a/api/worker.api.md b/api/worker.api.md index b9db1bb8..63a4c3ef 100644 --- a/api/worker.api.md +++ b/api/worker.api.md @@ -1053,7 +1053,7 @@ export function createAgentConfigOf(agent: SimpleAgent): CreateAgentConfig | und // @public export type CreateAgentInstructions = { instructions?: PerRun; - prompt?: undefined; + prompt?: PromptRefString; } | { instructions?: undefined; prompt?: PerRun; @@ -1154,6 +1154,7 @@ interface DefineToolOptions { name: string; needsApproval?: boolean | ((args: InferSchemaOutput, ctx: ApprovalCheckContext) => ApprovalOutcome | Promise); requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: InferSchemaOutput, sandbox: SandboxAdapter) => Promise; timeoutMs?: number; } @@ -1346,6 +1347,7 @@ const ERROR_CODES: { readonly LOUSHO_RUN_ALREADY_ITERATED: "Iterate an AgentRun once; call stream() again for a new run."; readonly LOUSHO_SANDBOX_PATH_DENIED: "Write under the sandbox's root (SubprocessSandbox({ root })), or pass the root that should contain the path."; readonly LOUSHO_SANDBOX_EGRESS_UNSUPPORTED: "Run on Docker Engine 25.0.5+ for Linux on this host (not Docker Desktop, rootless or a remote daemon), or use network: 'none'."; + readonly LOUSHO_SANDBOX_FAILED: "The error's `kind` says which boundary failed ('timeout', 'oom' or 'driver'); raise the limit in sandbox()/defineTool({ sandbox }), or check the driver."; readonly LOUSHO_AGENT_EXECUTION_FAILED: "Look at the `cause` for the underlying failure."; readonly LOUSHO_FLOW_EXECUTION_FAILED: "Look at the failing `step` and the `cause`."; readonly LOUSHO_VALIDATION_FAILED: "Fix the fields listed in `errors`."; @@ -2839,6 +2841,9 @@ interface Principal { // @public type PromptCachingOption = 'auto' | false; +// @public +type PromptRefString = `${string}@${number}` | `${string}@latest`; + // @public export interface PromptScheduleInput extends ScheduleBase { // (undocumented) @@ -3134,6 +3139,23 @@ interface SandboxAdapter { writeFile(path: string, content: string): Promise; } +// @public +interface SandboxCallOptions { + args?: unknown[]; +} + +// @public +interface SandboxDriver { + readonly name?: string; + run(code: string, opts?: SandboxCallOptions): Promise; +} + +// @public +type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +// @public +type SandboxDriverRef = SandboxDriverName | SandboxDriver; + // @public interface SandboxResult { // (undocumented) @@ -3956,6 +3978,7 @@ export interface ToolDescriptor { // (undocumented) needsApproval?: boolean | ApprovalPredicate; requiresSandbox?: boolean; + sandbox?: ToolSandboxOptions; sandboxExecute?: (args: unknown, sandbox: SandboxAdapter, ctx?: ToolExecutionContext) => Promise; tool: Tool; transient?: boolean; @@ -4065,6 +4088,13 @@ interface ToolResumeEvent extends AgentEventBase<'tool.resume'> { toolName: string; } +// @public +interface ToolSandboxOptions { + driver?: SandboxDriverRef; + memoryMb?: number; + timeoutMs?: number; +} + // @public interface ToolSearchOptions { contextWindow?: number; @@ -4093,9 +4123,7 @@ interface ToolStartEvent extends AgentEventBase<'tool.start'> { // @public interface TraceExporter { - // (undocumented) onSpanEnd(span: Span): void; - // (undocumented) onSpanStart(span: Span): void; } diff --git a/apps/agent-forge/server/__tests__/observeProxy.test.ts b/apps/agent-forge/server/__tests__/observeProxy.test.ts new file mode 100644 index 00000000..728993de --- /dev/null +++ b/apps/agent-forge/server/__tests__/observeProxy.test.ts @@ -0,0 +1,75 @@ +/** + * `/observe/**` -> the observe daemon: the studio server forwards the + * panels' GETs same-origin, because the daemon itself serves no CORS + * headers. A dead daemon answers 502 with the `lousho observe start` hint, + * and non-GET methods stay reporter-only (405). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as http from 'node:http'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { once } from 'node:events'; +import request from 'supertest'; +import { createApp } from '../app'; +import { RunManager } from '../runRegistry'; +import { FileCheckpointStore } from '../checkpointStore'; +import { FileApprovalStore } from '../approvalStore'; +import { createFsAgentStore } from '../../src/persistence/fsAgentStore'; + +describe('observe proxy', () => { + let baseDir: string; + let app: ReturnType; + let daemon: http.Server | undefined; + let daemonUrl: string; + const prevEnv = process.env.LOUSHO_OBSERVE_URL; + + beforeEach(async () => { + baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lou-observe-proxy-test-')); + const agentStore = createFsAgentStore(baseDir); + const runManager = new RunManager({ + baseDir, + checkpointStore: new FileCheckpointStore(baseDir), + approvalStore: new FileApprovalStore(baseDir), + loadSpec: (id) => agentStore.load(id), + saveSpec: (id, spec) => agentStore.save(id, spec), + }); + app = createApp({ agentStore, runManager, baseDir }); + + // A stand-in for `lousho observe`: echoes the path it was asked for. + daemon = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ echoed: req.url })); + }); + daemon.listen(0, '127.0.0.1'); + await once(daemon, 'listening'); + daemonUrl = `http://127.0.0.1:${(daemon.address() as { port: number }).port}`; + process.env.LOUSHO_OBSERVE_URL = daemonUrl; + }); + + afterEach(async () => { + if (prevEnv === undefined) delete process.env.LOUSHO_OBSERVE_URL; + else process.env.LOUSHO_OBSERVE_URL = prevEnv; + daemon?.close(); + fs.rmSync(baseDir, { recursive: true, force: true }); + }); + + it('forwards GETs under /observe to the daemon, prefix stripped', async () => { + const res = await request(app).get('/observe/api/v1/traces?status=error&limit=10'); + expect(res.status).toBe(200); + expect(res.body.echoed).toBe('/api/v1/traces?status=error&limit=10'); + }); + + it('answers 502 with the start hint when the daemon is down', async () => { + daemon?.close(); + daemon = undefined; + const res = await request(app).get('/observe/api/v1/agents'); + expect(res.status).toBe(502); + expect(res.body.error).toContain('lousho observe start'); + }); + + it('does not forward writes - Studio is read-only over the daemon API', async () => { + const res = await request(app).post('/observe/api/v1/ingest').send({ batch: [] }); + expect(res.status).toBe(405); + }); +}); diff --git a/apps/agent-forge/server/app.ts b/apps/agent-forge/server/app.ts index 8847947f..6499f183 100644 --- a/apps/agent-forge/server/app.ts +++ b/apps/agent-forge/server/app.ts @@ -17,6 +17,7 @@ * GET /runs/:id/history -> RunHistoryPayload * POST /runs/:id/fork -> body: ForkRunRequest; starts the fork, 202 ForkRunResponse * GET /runs/compare?a=&b= -> RunComparisonPayload (compareTrajectories) + * GET /observe/** -> proxied to the `lousho observe` daemon's read API (observeProxy.ts) * * `WS /agents/:id/stream` is wired separately in wsServer.ts (attached to * the same underlying http.Server, since `ws` needs the raw HTTP upgrade @@ -58,6 +59,7 @@ import { } from './traceStore'; import { DEPLOY_ADAPTERS, isDeployAdapter, runDeploy } from './deployRunner'; import { checkApiAccess, checkHostAndOrigin, type StudioAccessOptions } from './auth'; +import { registerObserveProxy } from './observeProxy'; export interface CreateAppOptions { agentStore: AgentStore; @@ -90,7 +92,7 @@ export interface CreateAppOptions { } /** Path prefixes of the API (everything except `/health` and the static client). */ -const API_PREFIXES = ['/agents', '/runs', '/settings', '/workspace']; +const API_PREFIXES = ['/agents', '/runs', '/settings', '/workspace', '/observe']; function asyncRoute(fn: (req: Request, res: Response) => Promise) { return (req: Request, res: Response, next: NextFunction) => { @@ -594,7 +596,7 @@ function registerStaticClient(app: Express, staticDir: string | undefined): void // `App.tsx` view state, but still refresh-safe) - serve `index.html` and // let the client app take over, the same as `vite preview`/most SPA // hosts do. - app.get(/^(?!\/health|\/agents|\/runs).*/, (_req, res) => { + app.get(/^(?!\/health|\/agents|\/runs|\/observe).*/, (_req, res) => { res.sendFile(indexHtml); }); } @@ -664,6 +666,10 @@ export function createApp({ registerProviderKeyRoutes(app, secrets); registerProfileRoutes(app, settings); registerDeployRoutes(app, agentStore, baseDir); + // The observe panels' read path: `/observe/**` -> the `lousho observe` + // daemon (see observeProxy.ts) - before the SPA fallback so a proxied + // GET never gets index.html. + registerObserveProxy(app); registerStaticClient(app, staticDir); // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/apps/agent-forge/server/observeProxy.ts b/apps/agent-forge/server/observeProxy.ts new file mode 100644 index 00000000..55832d52 --- /dev/null +++ b/apps/agent-forge/server/observeProxy.ts @@ -0,0 +1,50 @@ +/** + * `/observe/**` -> the observe daemon's read API (`src/observe/routes/*`). + * + * The daemon binds loopback (default `http://127.0.0.1:7777`) and sends no + * CORS headers, so the Studio panels can't fetch it cross-origin; the + * studio server forwards its GETs same-origin instead, keeping the + * "daemon owns the only db handle" rule - this proxy pipes, it never reads + * the file. `LOUSHO_OBSERVE_URL` repoints the upstream (another port, or a + * hosted collector implementing the same ingest/read API). + * + * Only GET is forwarded: Studio is a read surface; the ingest endpoint stays + * reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it carries the + * same per-launch token requirement as the rest of the studio API. + */ +import type { Express } from 'express'; + +const DEFAULT_OBSERVE_URL = 'http://127.0.0.1:7777'; +const PROXY_PREFIX = '/observe'; +const UPSTREAM_TIMEOUT_MS = 8000; + +function observeTarget(): string { + return (process.env.LOUSHO_OBSERVE_URL ?? DEFAULT_OBSERVE_URL).replace(/\/+$/, ''); +} + +export function registerObserveProxy(app: Express): void { + app.use(PROXY_PREFIX, async (req, res) => { + if (req.method !== 'GET') { + res.status(405).json({ error: 'The observe proxy only forwards GET reads.' }); + return; + } + const upstream = `${observeTarget()}${req.originalUrl.slice(PROXY_PREFIX.length)}`; + let response: Response; + try { + response = await fetch(upstream, { + method: 'GET', + headers: { accept: 'application/json' }, + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + } catch { + res + .status(502) + .json({ error: 'The observe daemon is unreachable - run `lousho observe start` to collect agent data.' }); + return; + } + res.status(response.status); + const contentType = response.headers.get('content-type'); + if (contentType) res.setHeader('content-type', contentType); + res.send(Buffer.from(await response.arrayBuffer())); + }); +} diff --git a/apps/agent-forge/src/App.tsx b/apps/agent-forge/src/App.tsx index ed80dc9c..170c124e 100644 --- a/apps/agent-forge/src/App.tsx +++ b/apps/agent-forge/src/App.tsx @@ -9,6 +9,7 @@ import { Inspector } from './components/Inspector'; import { BottomDrawer } from './components/BottomDrawer'; import { ErrorBoundary } from './components/ErrorBoundary'; import { NARROW_QUERY, useMediaQuery } from './components/useMediaQuery'; +import { ObserveView } from './observe/ObserveView'; /** * Below 860px the rail and the Inspector are slide-over drawers, @@ -46,7 +47,20 @@ function ConnectionBanner() { function Studio() { const panels = useSidePanels(); + const { view } = useAppState(); const shellClass = `shell${panels.open ? ` ${panels.open}-open` : ''}`; + // Observe mode: the daemon-backed panels take the whole content area + // (builder rail/canvas/inspector and the run drawer are build-mode chrome). + if (view === 'observe') { + return ( +
+

Agent Forge

+ + + +
+ ); + } return (

Agent Forge

diff --git a/apps/agent-forge/src/components/Topbar.tsx b/apps/agent-forge/src/components/Topbar.tsx index d7603ced..db299a21 100644 --- a/apps/agent-forge/src/components/Topbar.tsx +++ b/apps/agent-forge/src/components/Topbar.tsx @@ -81,6 +81,35 @@ function EnvSelect() { ); } +/** + * Build/Observe surface switcher. "Observe" swaps the builder shell for the + * observability panels (Traces/Insights/Monitoring/Scores) that read the + * `lousho observe` daemon - a separate data source from the studio server. + */ +function ViewToggle() { + const { view, setView } = useAppState(); + return ( +
+ + +
+ ); +} + function DebugToggle() { const { debugMode, setDebugMode, setDrawerTab } = useAppState(); return ( @@ -285,7 +314,7 @@ interface TopbarProps { } function NarrowTopbar({ openPanel, onTogglePanel }: Omit) { - const { spec, dirty, runStatus, debugMode, setDebugMode, setDrawerTab } = useAppState(); + const { spec, dirty, runStatus, debugMode, setDebugMode, setDrawerTab, view, setView } = useAppState(); const { actionError, handleRun, handleStop, handleSave, handleImport } = useRunActions(); const { fileInput, openImport, exportSpec } = useSpecFiles(handleImport); const toggle = onTogglePanel ?? (() => undefined); @@ -305,6 +334,11 @@ function NarrowTopbar({ openPanel, onTogglePanel }: Omit) { label: 'Import', onSelect: openImport }, { label: 'Export', onSelect: exportSpec }, { label: 'Save', onSelect: () => void handleSave() }, + // Last: the menu focuses its first item on open (e2e expects Debug). + { + label: view === 'observe' ? 'Build surface' : 'Observe surface', + onSelect: () => setView(view === 'observe' ? 'build' : 'observe'), + }, ]; return ( @@ -365,6 +399,8 @@ function WideTopbar() {
+ + diff --git a/apps/agent-forge/src/components/layout.css b/apps/agent-forge/src/components/layout.css index 794fe969..46d8e9f2 100644 --- a/apps/agent-forge/src/components/layout.css +++ b/apps/agent-forge/src/components/layout.css @@ -11,6 +11,29 @@ display: grid; grid-template-rows: 48px 1fr 240px; } +/* Observe mode: the panels take the whole content area, no run drawer. */ +.app-observe { + grid-template-rows: 48px 1fr; +} + +/* Build/Observe surface switcher (Topbar.tsx's ViewToggle). */ +.view-toggle { + display: flex; + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; + flex: none; +} +.view-toggle .btn { + border: none; + border-radius: 0; + padding: 5px 10px; + font-size: 11.5px; +} +.view-toggle .btn.active { + background: var(--accent-soft); + color: var(--accent); +} /* read by screen readers, not drawn (the page h1, status prefixes). */ .visually-hidden { diff --git a/apps/agent-forge/src/observe/FleetPanel.tsx b/apps/agent-forge/src/observe/FleetPanel.tsx new file mode 100644 index 00000000..99e88d98 --- /dev/null +++ b/apps/agent-forge/src/observe/FleetPanel.tsx @@ -0,0 +1,94 @@ +import { useEffect, useState } from 'react'; +import { useAgents, useInsights } from './useObserve'; +import type { ObserveAgentRow, ObserveAgentInsights } from './types'; +import { fmtAge, fmtInt, fmtPercent } from './format'; +import { ObserveDaemonDown, ObserveError, ObserveStatusPill } from './chrome'; +import { ObserveDaemonDownError } from './observeClient'; + +/** Fleet freshness poll - the Monitoring surface is the "is my fleet up" pane, so it self-refreshes. */ +const POLL_MS = 15_000; + +function usePoll(ms: number): number { + const [tick, setTick] = useState(0); + useEffect(() => { + const timer = setInterval(() => setTick((t) => t + 1), ms); + return () => clearInterval(timer); + }, [ms]); + return tick; +} + +function agentLabel(agent: ObserveAgentRow): string { + return agent.name ?? agent.agent_id.slice(0, 8); +} + +/** + * `GET /api/v1/agents` + `GET /api/v1/insights?days=1` - the fleet registry + * (name, project, derived status, last heartbeat) joined with today's run + * and error figures. "View traces" drills into the Traces tab scoped to + * that agent via ObserveView's shared filter state. + */ +export function FleetPanel({ onShowTraces }: { onShowTraces: (agentId: string) => void }) { + const tick = usePoll(POLL_MS); + const agents = useAgents(undefined, tick); + const today = useInsights({ days: 1 }, tick); + const todayByAgent = new Map((today.data?.agents ?? []).map((a) => [a.agent_id, a])); + + if (agents.error instanceof ObserveDaemonDownError) return ; + + const rows = agents.data ?? []; + return ( +
+ + {agents.loading && agents.data === undefined ? ( +
Loading fleet...
+ ) : rows.length === 0 ? ( +
+

No agents are registered yet. An agent registers on its first emit when LOUSHO_OBSERVE=1.

+
+ ) : ( +
+ + + + + + + + + + + + + + {rows.map((agent) => { + const stats = todayByAgent.get(agent.agent_id); + return ( + + + + + + + + + + ); + })} + +
AgentProjectStatusLast heartbeatRuns todayError rate
+ {agentLabel(agent)} + {agent.version && v{agent.version}} + {agent.project ?? '-'} + + + {fmtAge(agent.heartbeat_age_ms)} + {stats ? fmtInt(stats.runs) : '-'}{stats ? fmtPercent(stats.error_rate) : '-'} + +
+
+ )} +
+ ); +} diff --git a/apps/agent-forge/src/observe/InsightsPanel.tsx b/apps/agent-forge/src/observe/InsightsPanel.tsx new file mode 100644 index 00000000..671b8791 --- /dev/null +++ b/apps/agent-forge/src/observe/InsightsPanel.tsx @@ -0,0 +1,154 @@ +import { useMemo, useState } from 'react'; +import { useInsights } from './useObserve'; +import type { ObserveAgentInsights, ObserveRollupRow } from './types'; +import { fmtCost, fmtDuration, fmtInt, fmtPercent } from './format'; +import { ObserveDaemonDown, ObserveError } from './chrome'; +import { ObserveDaemonDownError } from './observeClient'; + +const DAY_OPTIONS = [1, 7, 30] as const; +const METRICS = [ + { id: 'runs', label: 'Runs' }, + { id: 'errors', label: 'Errors' }, + { id: 'cost_usd', label: 'Cost ($)' }, + { id: 'input_tokens', label: 'Input tokens' }, + { id: 'output_tokens', label: 'Output tokens' }, +] as const; +type MetricId = (typeof METRICS)[number]['id']; + +function fmtMetric(metric: MetricId, value: number): string { + return metric === 'cost_usd' ? fmtCost(value) : fmtInt(value); +} + +/** One agent's aggregate card: runs/day, p50/p95, cost, error rate - the numbers the spec's Insights section names. */ +function AgentCard({ agent }: { agent: ObserveAgentInsights }) { + return ( +
+
+ {agent.name ?? agent.agent_id.slice(0, 8)} + {agent.project && {agent.project}} +
+
+
+ {agent.runs_per_day.toFixed(1)} + runs/day +
+
+ {fmtDuration(agent.p50_latency_ms)} + p50 +
+
+ {fmtDuration(agent.p95_latency_ms)} + p95 +
+
+ {fmtCost(agent.cost_usd)} + cost +
+
+ 0 ? ' insight-bad' : ''}`}> + {fmtPercent(agent.error_rate)} + + error rate +
+
+ {fmtInt(agent.input_tokens + agent.output_tokens)} + tokens +
+
+
+ ); +} + +interface Bucket { + start: number; + value: number; +} + +/** Sum every agent's rollup rows into one (bucket_start -> value) series for `metric`. */ +function metricBuckets(series: ObserveRollupRow[], metric: MetricId): Bucket[] { + const byBucket = new Map(); + for (const row of series) { + if (row.metric !== metric) continue; + byBucket.set(row.bucket_start, (byBucket.get(row.bucket_start) ?? 0) + row.value); + } + return [...byBucket.entries()].sort(([a], [b]) => a - b).map(([start, value]) => ({ start, value })); +} + +/** Minimal bar chart - plain divs, no chart dependency. */ +function MetricChart({ buckets, metric }: { buckets: Bucket[]; metric: MetricId }) { + const max = Math.max(...buckets.map((b) => b.value), 0); + if (buckets.length === 0) return

No rollup rows in this window yet.

; + return ( +
+ {buckets.map((bucket) => ( +
+ ))} +
+ ); +} + +/** + * `GET /api/v1/insights` - per-agent aggregate cards plus the rollup time + * series. Aggregates come from the daemon's `rollups` table; the panel only + * picks the window (days) and which metric the chart draws. + */ +export function InsightsPanel() { + const [days, setDays] = useState(7); + const [metric, setMetric] = useState('runs'); + const { data, error, loading } = useInsights({ days, granularity: days <= 1 ? 'hour' : 'day' }); + const buckets = useMemo(() => metricBuckets(data?.series ?? [], metric), [data, metric]); + + if (error instanceof ObserveDaemonDownError) return ; + + return ( +
+
+ + +
+ + {loading && data === undefined ? ( +
Loading insights...
+ ) : (data?.agents.length ?? 0) === 0 ? ( +
+

No rollup data in this window - Insights fills in as agents report runs.

+
+ ) : ( + <> +
+ {data!.agents.map((agent) => ( + + ))} +
+
+ {METRICS.find((m) => m.id === metric)?.label} per {days <= 1 ? 'hour' : 'day'} (all agents) +
+ + + )} +
+ ); +} diff --git a/apps/agent-forge/src/observe/ObserveView.tsx b/apps/agent-forge/src/observe/ObserveView.tsx new file mode 100644 index 00000000..09bb1879 --- /dev/null +++ b/apps/agent-forge/src/observe/ObserveView.tsx @@ -0,0 +1,98 @@ +import './observe.css'; +import { useState } from 'react'; +import { tabIds, tabListKeyDown } from '../components/tabs'; +import { useDaemonHealth } from './useObserve'; +import { ObserveDaemonDown } from './chrome'; +import type { TraceListFilter } from './observeClient'; +import { TracesPanel } from './TracesPanel'; +import { TraceDetailView } from './TraceDetailView'; +import { InsightsPanel } from './InsightsPanel'; +import { FleetPanel } from './FleetPanel'; +import { ScoresPanel } from './ScoresPanel'; + +const TABS = [ + { id: 'traces', label: 'Traces' }, + { id: 'insights', label: 'Insights' }, + { id: 'fleet', label: 'Monitoring' }, + { id: 'scores', label: 'Scores' }, +] as const; +type ObserveTab = (typeof TABS)[number]['id']; +const TAB_IDS: ObserveTab[] = TABS.map((t) => t.id); + +/** + * The observability surface - Traces, Insights, Monitoring (fleet) and + * Scores - served by the observe daemon's read API through the same-origin + * `/observe` proxy. Cross-panel drill-down (fleet row -> that agent's + * traces) is just shared state here: `tracesFilter` is owned at the top so + * another tab can pre-fill it. + */ +export function ObserveView() { + const health = useDaemonHealth(); + const [tab, setTab] = useState('traces'); + const [tracesFilter, setTracesFilter] = useState({}); + const [traceId, setTraceId] = useState(undefined); + + // Monitoring's "view traces" click-through: same table, scoped to the agent. + const drillToAgent = (agentId: string) => { + setTracesFilter((filter) => ({ ...filter, agent_id: agentId })); + setTraceId(undefined); + setTab('traces'); + }; + const drillToTrace = (id: string) => { + setTraceId(id); + setTab('traces'); + }; + + let body: React.ReactNode; + if (health.down) body = ; + else if (tab === 'traces' && traceId !== undefined) + body = setTraceId(undefined)} />; + else if (tab === 'traces') + body = ( + { + setTracesFilter(filter); + setTraceId(undefined); + }} + onOpenTrace={setTraceId} + /> + ); + else if (tab === 'insights') body = ; + else if (tab === 'fleet') body = ; + else body = ; + + const ids = tabIds('observe', tab); + return ( +
+
+ {TABS.map((t) => { + const selected = tab === t.id; + const tids = tabIds('observe', t.id); + return ( + + ); + })} +
+
+ {body} +
+
+ ); +} diff --git a/apps/agent-forge/src/observe/ScoresPanel.tsx b/apps/agent-forge/src/observe/ScoresPanel.tsx new file mode 100644 index 00000000..2d8ed95d --- /dev/null +++ b/apps/agent-forge/src/observe/ScoresPanel.tsx @@ -0,0 +1,163 @@ +import { useMemo, useState } from 'react'; +import { useAgents, useScores } from './useObserve'; +import type { ObserveScoreRow } from './types'; +import { fmtDateTime, fmtInt, fmtPercent } from './format'; +import { AgentSelect, ObserveDaemonDown, ObserveError } from './chrome'; +import { ObserveDaemonDownError } from './observeClient'; + +const LIMIT = 200; +const DAY_MS = 86_400_000; +/** A numeric score at/above this counts as a pass for the per-eval trend (eval scores land 0..1). */ +const PASS_AT = 0.5; + +interface EvalTrend { + name: string; + count: number; + mean: number | null; + passRate: number | null; + /** Daily pass fractions, oldest first - the trend strip. */ + days: { start: number; passRate: number; count: number }[]; +} + +/** Group score rows by eval `name`: count, mean, overall pass rate and the per-day pass-rate strip. */ +function evalTrends(scores: ObserveScoreRow[]): EvalTrend[] { + const byName = new Map(); + for (const score of scores) { + const list = byName.get(score.name) ?? []; + list.push(score); + byName.set(score.name, list); + } + const trends: EvalTrend[] = []; + for (const [name, rows] of byName) { + const scored = rows.filter((row) => row.value !== null); + const mean = scored.length ? scored.reduce((sum, row) => sum + row.value!, 0) / scored.length : null; + const passRate = scored.length ? scored.filter((row) => row.value! >= PASS_AT).length / scored.length : null; + const byDay = new Map(); + for (const row of scored) { + const day = row.created_at - (row.created_at % DAY_MS); + const bucket = byDay.get(day) ?? { pass: 0, count: 0 }; + bucket.count += 1; + if (row.value! >= PASS_AT) bucket.pass += 1; + byDay.set(day, bucket); + } + const days = [...byDay.entries()] + .sort(([a], [b]) => a - b) + .map(([start, bucket]) => ({ start, passRate: bucket.pass / bucket.count, count: bucket.count })); + trends.push({ name, count: rows.length, mean, passRate, days }); + } + return trends.sort((a, b) => b.count - a.count); +} + +function TrendStrip({ trend }: { trend: EvalTrend }) { + return ( +
+
+ {trend.name} + + {fmtInt(trend.count)} scores · mean {trend.mean === null ? '-' : trend.mean.toFixed(2)} · pass{' '} + {trend.passRate === null ? '-' : fmtPercent(trend.passRate)} + +
+
+ {trend.days.map((day) => ( +
= PASS_AT ? ' pass' : ' fail'}`} + title={`${new Date(day.start).toLocaleDateString()} - ${fmtPercent(day.passRate)} pass (${day.count})`} + style={{ height: `${Math.max(15, day.passRate * 100)}%` }} + /> + ))} +
+
+ ); +} + +/** + * `GET /api/v1/scores` - eval/judge/human score rows with the daemon's + * `name`/`agent_id`/`trace_id` filters, plus a per-eval-name pass-rate + * trend computed over the returned window. A score id links back to its + * trace (ObserveView opens {@link TraceDetailView}). + */ +export function ScoresPanel({ onOpenTrace }: { onOpenTrace: (traceId: string) => void }) { + const [name, setName] = useState(''); + const [agentId, setAgentId] = useState(''); + const agents = useAgents(); + const { data, error, loading } = useScores({ name: name || undefined, agent_id: agentId || undefined, limit: LIMIT }); + + const trends = useMemo(() => evalTrends(data ?? []), [data]); + const names = useMemo(() => [...new Set((data ?? []).map((s) => s.name))].sort(), [data]); + + if (error instanceof ObserveDaemonDownError) return ; + + return ( +
+
+ + +
+ + {loading && data === undefined ? ( +
Loading scores...
+ ) : (data?.length ?? 0) === 0 ? ( +
+

No scores yet - eval results land here once a run's evals emit score events.

+
+ ) : ( + <> + {trends.length > 0 && ( +
+ {trends.map((trend) => ( + + ))} +
+ )} +
+ + + + + + + + + + + + + {data!.map((score) => ( + + + + + + + + + ))} + +
WhenEvalValueSourceTraceComment
{fmtDateTime(score.created_at)}{score.name} + {score.value === null ? '-' : score.value} + {score.source} + + {score.comment ?? ''}
+
+ + )} +
+ ); +} diff --git a/apps/agent-forge/src/observe/TraceDetailView.tsx b/apps/agent-forge/src/observe/TraceDetailView.tsx new file mode 100644 index 00000000..d6952b9c --- /dev/null +++ b/apps/agent-forge/src/observe/TraceDetailView.tsx @@ -0,0 +1,209 @@ +import { useMemo, useState } from 'react'; +import { JsonTree } from '../components/debug/JsonTree'; +import { useTraceDetail } from './useObserve'; +import { flattenObservationTree, nestObservations, traceBounds, type ObservationNode } from './tree'; +import type { ObserveObservationRow, ObserveScoreRow } from './types'; +import { fmtCost, fmtDateTime, fmtDuration, fmtInt } from './format'; +import { ObserveError, ObserveDaemonDown, ObserveStatusPill } from './chrome'; +import { ObserveDaemonDownError } from './observeClient'; + +const KIND_COLOR: Record = { + span: 'var(--info)', + generation: 'var(--accent)', + tool: 'var(--warning)', + event: 'var(--text-faint)', +}; + +function classNames(...names: (string | false)[]): string { + return names.filter(Boolean).join(' '); +} + +function obsDuration(row: ObserveObservationRow): number | undefined { + return row.end === null ? undefined : row.end - row.start; +} + +function ObsRow({ + node, + minStart, + totalMs, + selected, + onSelect, +}: { + node: ObservationNode; + minStart: number; + totalMs: number; + selected: boolean; + onSelect: () => void; +}) { + const { row } = node; + const startPct = ((row.start - minStart) / totalMs) * 100; + const duration = obsDuration(row); + const widthPct = Math.max(0.5, ((duration ?? 0) / totalMs) * 100); + const failed = row.status === 'error'; + return ( +
e.key === 'Enter' && onSelect()} + title={row.name ?? row.obs_id} + > + + + {row.kind} + + + + {duration === undefined ? 'running' : `${Math.round(duration)}ms`} +
+ ); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+
{label}
+
{children}
+
+ ); +} + +/** The right-hand inspector for the selected observation: io, usage/cost (flags kept honest), attrs. */ +function ObservationInspector({ row }: { row: ObserveObservationRow }) { + return ( +
+
+ + {row.kind} + + {row.name ?? row.obs_id} + +
+
+ {row.model && {row.model}} + {fmtDateTime(row.start)} + {row.end === null ? 'still running' : fmtDuration(row.end - row.start)} + {(row.usage_input_tokens !== null || row.usage_output_tokens !== null) && ( + + {fmtInt(row.usage_input_tokens)} in / {fmtInt(row.usage_output_tokens)} out + {row.usage_cached_tokens ? ` (${fmtInt(row.usage_cached_tokens)} cached)` : ''} + {row.usage_reasoning_tokens ? ` (${fmtInt(row.usage_reasoning_tokens)} reasoning)` : ''} + {row.usage_estimated ? ' · estimated' : ''} + + )} + {row.cost_usd !== null && ( + + {fmtCost(row.cost_usd)} + {row.cost_partial ? ' (partial)' : ''} + + )} + {row.input !== null && ( + + 
{row.input}
+ + )} + {row.output !== null && ( + + 
{row.output}
+ {row.truncated &&
Truncated by the reporter at ~64KB.
} + + )} + {row.attrs && ( + + + + )} +
+
+ ); +} + +function ScoresBlock({ scores }: { scores: ObserveScoreRow[] }) { + if (scores.length === 0) return null; + return ( +
+
Scores
+ + + + + + + + + + + {scores.map((score) => ( + + + + + + + ))} + +
NameValueSourceComment
{score.name}{score.value === null ? '-' : score.value}{score.source}{score.comment ?? ''}
+
+ ); +} + +/** + * `GET /api/v1/traces/:id` - flat observation rows nested client-side by + * `parent_obs_id` ({@link nestObservations}), drawn as the same latency + * waterfall the live Trace tab uses, plus a per-observation inspector and + * the trace's score rows. + */ +export function TraceDetailView({ traceId, onBack }: { traceId: string; onBack: () => void }) { + const { data, error, loading } = useTraceDetail(traceId); + const [selectedObsId, setSelectedObsId] = useState(undefined); + + const flat = useMemo(() => { + if (!data) return []; + return flattenObservationTree(nestObservations(data.observations)); + }, [data]); + const bounds = useMemo(() => traceBounds(data?.observations ?? []), [data]); + const totalMs = Math.max(1, bounds.maxEnd - bounds.minStart); + const selected = flat.find((node) => node.row.obs_id === selectedObsId); + + if (loading && data === undefined) return
Loading trace...
; + if (error) return error instanceof ObserveDaemonDownError ? : ; + if (!data) return

No trace '{traceId}'.

; + + return ( +
+
+ + + {data.trace_id.slice(0, 16)} + + + {flat.length} observations · {data.scores.length} scores + +
+
+
+ {flat.map((node) => ( + setSelectedObsId(node.row.obs_id === selectedObsId ? undefined : node.row.obs_id)} + /> + ))} + +
+ {selected && } +
+
+ ); +} diff --git a/apps/agent-forge/src/observe/TracesPanel.tsx b/apps/agent-forge/src/observe/TracesPanel.tsx new file mode 100644 index 00000000..fd92491c --- /dev/null +++ b/apps/agent-forge/src/observe/TracesPanel.tsx @@ -0,0 +1,161 @@ +import { useEffect, useState } from 'react'; +import type { TraceListFilter } from './observeClient'; +import { useAgents, useTraces } from './useObserve'; +import type { ObserveTraceRow } from './types'; +import { fmtCost, fmtDateTime, fmtDuration, fmtInt } from './format'; +import { AgentSelect, ObserveError, ObserveStatusPill } from './chrome'; + +const PAGE_SIZE = 50; +const STATUS_OPTIONS = ['', 'ok', 'error', 'interrupted'] as const; + +function FilterBar({ + filter, + onChange, +}: { + filter: TraceListFilter; + onChange: (filter: TraceListFilter) => void; +}) { + const agents = useAgents(); + const projects = [...new Set((agents.data ?? []).map((a) => a.project).filter((p): p is string => p !== null))].sort(); + const set = (patch: Partial) => onChange({ ...filter, ...patch }); + + return ( +
+ + + + {agents.error && } +
+ ); +} + +function TraceRow({ trace, onOpen }: { trace: ObserveTraceRow; onOpen: () => void }) { + const duration = trace.end - trace.start; + return ( + e.key === 'Enter' && onOpen()}> + + {trace.name ?? {trace.trace_id.slice(0, 12)}} + + {fmtDateTime(trace.start)} + {fmtDuration(duration)} + + + + {fmtInt(trace.observations)} + + {fmtInt(trace.input_tokens + trace.output_tokens)} + + {fmtCost(trace.cost_usd)} + + ); +} + +/** + * `GET /api/v1/traces` - the paginated, filterable trace list (time-ordered, + * newest first). Clicking a row hands the trace id to ObserveView, which + * swaps in {@link TraceDetailView}. + */ +export function TracesPanel({ + filter, + onFilterChange, + onOpenTrace, +}: { + filter: TraceListFilter; + onFilterChange: (filter: TraceListFilter) => void; + onOpenTrace: (traceId: string) => void; +}) { + const [offset, setOffset] = useState(0); + // Any filter edit restarts pagination at the first page. + useEffect(() => setOffset(0), [JSON.stringify(filter)]); + const query = useTraces({ ...filter, limit: PAGE_SIZE, offset }); + const { data, error, loading } = query; + const total = data?.total ?? 0; + const page = Math.floor(offset / PAGE_SIZE) + 1; + const pages = Math.max(1, Math.ceil(total / PAGE_SIZE)); + + return ( +
+ + + {loading && data === undefined ? ( +
Loading traces...
+ ) : (data?.traces.length ?? 0) === 0 ? ( +
+

No traces match this filter. Traces appear once an agent with LOUSHO_OBSERVE=1 finishes a run.

+
+ ) : ( + <> +
+ + + + + + + + + + + + + + {data!.traces.map((trace) => ( + onOpenTrace(trace.trace_id)} /> + ))} + +
NameStartedDurationStatusSpansTokensCost
+
+
+ + + Page {page} of {pages} · {fmtInt(total)} traces + + +
+ + )} +
+ ); +} diff --git a/apps/agent-forge/src/observe/__tests__/ObserveView.test.tsx b/apps/agent-forge/src/observe/__tests__/ObserveView.test.tsx new file mode 100644 index 00000000..540291a2 --- /dev/null +++ b/apps/agent-forge/src/observe/__tests__/ObserveView.test.tsx @@ -0,0 +1,68 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { ObserveView } from '../ObserveView'; + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +async function flush() { + await act(async () => { + await Promise.resolve(); + }); +} + +describe('ObserveView', () => { + let root: Root | undefined; + let container: HTMLDivElement | undefined; + + afterEach(() => { + act(() => root?.unmount()); + container?.remove(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('shows the `lousho observe start` hint when the daemon is unreachable', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new TypeError('fetch failed'); + }) + ); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + await act(async () => { + root!.render(); + }); + await flush(); + expect(container.textContent).toContain('lousho observe start'); + expect(container.querySelector('[role="status"]')).not.toBeNull(); + }); + + it('renders the trace list once the daemon answers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith('/health')) { + return new Response(JSON.stringify({ service: 'lousho-observe', ok: true }), { status: 200 }); + } + if (url.includes('/api/v1/agents')) { + return new Response(JSON.stringify({ agents: [] }), { status: 200 }); + } + return new Response(JSON.stringify({ traces: [], total: 0, limit: 50, offset: 0 }), { status: 200 }); + }) + ); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + await act(async () => { + root!.render(); + }); + await flush(); + await flush(); + expect(container.textContent).not.toContain('the observe daemon is not reachable'); + expect(container.textContent).toContain('No traces match'); + }); +}); diff --git a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts new file mode 100644 index 00000000..2a2a0d4d --- /dev/null +++ b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, vi } from 'vitest'; +import { ObserveApiError, ObserveClient, ObserveDaemonDownError } from '../observeClient'; + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); +} + +function clientWith(fetchFn: typeof fetch): ObserveClient { + return new ObserveClient({ baseUrl: '/observe', fetchFn }); +} + +describe('ObserveClient', () => { + it('builds the traces URL with filter params, skipping empty ones', async () => { + const fetchFn = vi.fn(async () => jsonResponse(200, { traces: [], total: 0, limit: 50, offset: 0 })); + const client = clientWith(fetchFn as typeof fetch); + await client.listTraces({ agent_id: 'a1', status: 'error', limit: 25, offset: 50, project: '' }); + expect(fetchFn).toHaveBeenCalledWith('/observe/api/v1/traces?agent_id=a1&status=error&limit=25&offset=50', expect.anything()); + }); + + it('maps a proxy 502 and a network failure to ObserveDaemonDownError', async () => { + const via502 = clientWith((async () => jsonResponse(502, { error: 'unreachable' })) as typeof fetch); + await expect(via502.listTraces()).rejects.toBeInstanceOf(ObserveDaemonDownError); + const viaRefused = clientWith((async () => { + throw new TypeError('fetch failed'); + }) as typeof fetch); + await expect(viaRefused.listTraces()).rejects.toBeInstanceOf(ObserveDaemonDownError); + }); + + it('maps a daemon {error, code} body to ObserveApiError with its message', async () => { + const client = clientWith((async () => jsonResponse(400, { error: 'bad filter', code: 'LOUSHO_REQUEST_INVALID' })) as typeof fetch); + await expect(client.listTraces()).rejects.toMatchObject({ name: 'ObserveApiError', status: 400, message: 'bad filter' }); + await expect(client.listTraces()).rejects.toBeInstanceOf(ObserveApiError); + }); + + it('health() answers the health body only for a real lousho-observe marker', async () => { + const real = clientWith((async () => jsonResponse(200, { service: 'lousho-observe', ok: true, version: '1' })) as typeof fetch); + expect(await real.health()).toMatchObject({ service: 'lousho-observe' }); + const squatter = clientWith((async () => jsonResponse(200, { service: 'something-else', ok: true })) as typeof fetch); + expect(await squatter.health()).toBeUndefined(); + const down = clientWith((async () => { + throw new TypeError('fetch failed'); + }) as typeof fetch); + expect(await down.health()).toBeUndefined(); + }); + + it('traceDetail returns undefined on the daemon\'s 404 instead of throwing', async () => { + const client = clientWith((async () => jsonResponse(404, { error: "No trace 'x'.", code: 'LOUSHO_REQUEST_INVALID' })) as typeof fetch); + expect(await client.traceDetail('x')).toBeUndefined(); + }); + + it('sends the studio token only on the same-origin proxy base', async () => { + window.sessionStorage.setItem('lousho-studio-token', 'tok-123'); + try { + const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => { + void init; + return jsonResponse(200, { agents: [] }); + }); + const sameOrigin = new ObserveClient({ baseUrl: '/observe', fetchFn: fetchFn as typeof fetch }); + await sameOrigin.listAgents(); + const headers = new Headers((fetchFn.mock.calls[0][1] as RequestInit).headers); + expect(headers.get('x-lousho-studio-token')).toBe('tok-123'); + + const remote = new ObserveClient({ baseUrl: 'https://collector.example', fetchFn: fetchFn as typeof fetch }); + await remote.listAgents(); + const remoteHeaders = new Headers((fetchFn.mock.calls[1][1] as RequestInit).headers); + expect(remoteHeaders.get('x-lousho-studio-token')).toBeNull(); + } finally { + window.sessionStorage.removeItem('lousho-studio-token'); + } + }); +}); diff --git a/apps/agent-forge/src/observe/__tests__/tree.test.ts b/apps/agent-forge/src/observe/__tests__/tree.test.ts new file mode 100644 index 00000000..97d1090f --- /dev/null +++ b/apps/agent-forge/src/observe/__tests__/tree.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect } from 'vitest'; +import { flattenObservationTree, nestObservations, traceBounds } from '../tree'; +import type { ObserveObservationRow } from '../types'; + +let seq = 0; +function row(patch: Partial): ObserveObservationRow { + seq += 1; + return { + obs_id: `obs-${seq}`, + trace_id: 'trace-1', + session_id: null, + agent_id: 'agent-1', + parent_obs_id: null, + kind: 'span', + name: null, + start: 0, + end: null, + status: 'ok', + input: null, + output: null, + model: null, + usage_input_tokens: null, + usage_output_tokens: null, + usage_cached_tokens: null, + usage_reasoning_tokens: null, + cost_usd: null, + cost_partial: null, + usage_estimated: null, + truncated: false, + attrs: null, + ...patch, + }; +} + +describe('nestObservations', () => { + it('nests children under their parent_obs_id, roots sorted by start', () => { + const root = row({ obs_id: 'root', start: 100 }); + const childB = row({ obs_id: 'child-b', parent_obs_id: 'root', start: 300 }); + const childA = row({ obs_id: 'child-a', parent_obs_id: 'root', start: 200 }); + const grand = row({ obs_id: 'grand', parent_obs_id: 'child-a', start: 250 }); + // Rows arrive flat ordered by start, but order must not matter to the tree. + const tree = nestObservations([root, childA, grand, childB]); + expect(tree.map((n) => n.row.obs_id)).toEqual(['root']); + expect(tree[0].children.map((n) => n.row.obs_id)).toEqual(['child-a', 'child-b']); + expect(tree[0].children[0].children.map((n) => n.row.obs_id)).toEqual(['grand']); + expect(grand && tree[0].children[0].children[0].depth).toBe(2); + }); + + it('keeps multiple roots in start order', () => { + const later = row({ obs_id: 'later', start: 500 }); + const earlier = row({ obs_id: 'earlier', start: 100 }); + expect(nestObservations([later, earlier]).map((n) => n.row.obs_id)).toEqual(['earlier', 'later']); + }); + + it('hoists orphans whose parent_obs_id is missing instead of dropping them', () => { + const root = row({ obs_id: 'root', start: 0 }); + const orphan = row({ obs_id: 'orphan', parent_obs_id: 'gone', start: 50 }); + const tree = nestObservations([root, orphan]); + expect(tree.map((n) => n.row.obs_id)).toEqual(['root', 'orphan']); + expect(tree[1].depth).toBe(0); + }); + + it('treats a self-parent as a root and survives parent/child cycles', () => { + const selfish = row({ obs_id: 'self', parent_obs_id: 'self', start: 0 }); + const a = row({ obs_id: 'a', parent_obs_id: 'b', start: 10 }); + const b = row({ obs_id: 'b', parent_obs_id: 'a', start: 20 }); + const tree = nestObservations([selfish, a, b]); + // Both rows are reachable somewhere at root level - nothing is dropped. + const flat = flattenObservationTree(tree); + expect(flat.map((n) => n.row.obs_id).sort()).toEqual(['a', 'b', 'self']); + }); + + it('flattenObservationTree is depth-first with depth values', () => { + const root = row({ obs_id: 'root', start: 0 }); + const child = row({ obs_id: 'child', parent_obs_id: 'root', start: 10 }); + const sibling = row({ obs_id: 'sibling', start: 20 }); + const flat = flattenObservationTree(nestObservations([sibling, child, root])); + expect(flat.map((n) => [n.row.obs_id, n.depth])).toEqual([ + ['root', 0], + ['child', 1], + ['sibling', 0], + ]); + }); +}); + +describe('traceBounds', () => { + it('spans min start to max end, counting unfinished rows to now', () => { + const bounds = traceBounds( + [row({ start: 100, end: 400 }), row({ start: 200, end: null })], + 1000 + ); + expect(bounds).toEqual({ minStart: 100, maxEnd: 1000 }); + }); + + it('is degenerate-safe on empty input', () => { + expect(traceBounds([])).toEqual({ minStart: 0, maxEnd: 1 }); + }); +}); diff --git a/apps/agent-forge/src/observe/chrome.tsx b/apps/agent-forge/src/observe/chrome.tsx new file mode 100644 index 00000000..24b5c4a3 --- /dev/null +++ b/apps/agent-forge/src/observe/chrome.tsx @@ -0,0 +1,81 @@ +/** Shared bits of the observe panels' chrome: the daemon-down empty state, inline API errors, status pills, filter controls. */ + +import type { ObserveAgentRow } from './types'; + +/** The "All agents" filter select the panels share. */ +export function AgentSelect({ + agents, + value, + onChange, +}: { + agents: ObserveAgentRow[] | undefined; + value: string; + onChange: (agentId: string) => void; +}) { + return ( + + ); +} + +/** + * The "daemon not running" state every panel shares: the hint is the + * command, not a dead error, and the health probe keeps polling so the + * panels swap in by themselves once `lousho observe start` runs. + */ +export function ObserveDaemonDown() { + return ( +
+

No data yet - the observe daemon is not reachable.

+

+ Run lousho observe start, then set LOUSHO_OBSERVE=1 on your agents; their traces, + scores and heartbeats show up here automatically. +

+
+ ); +} + +/** An API failure that is not "daemon down" (a 400/404/500 answer) - shown inline so the panel chrome stays put. */ +export function ObserveError({ error }: { error: Error | undefined }) { + if (!error) return null; + return ( +
+ {error.message} +
+ ); +} + +/** `ok`/`error`/`interrupted`/`healthy`/… -> the status-pill colour classes shared with the rest of the app. */ +function statusClass(status: string | null | undefined): string { + switch (status) { + case 'ok': + case 'healthy': + return 'status-ok'; + case 'error': + case 'down': + return 'status-error'; + case 'interrupted': + case 'stale': + return 'status-interrupted'; + case 'running': + return 'status-running'; + default: + return 'status-idle'; + } +} + +export function ObserveStatusPill({ status }: { status: string | null | undefined }) { + const label = status ?? 'unknown'; + return ( + + + ); +} diff --git a/apps/agent-forge/src/observe/format.ts b/apps/agent-forge/src/observe/format.ts new file mode 100644 index 00000000..6760e30c --- /dev/null +++ b/apps/agent-forge/src/observe/format.ts @@ -0,0 +1,46 @@ +/** Small display formatters shared by the observe panels (all epoch-ms inputs). */ + +/** `1_234` grouped integer, `-` for null/undefined. */ +export function fmtInt(value: number | null | undefined): string { + if (value === null || value === undefined || !Number.isFinite(value)) return '-'; + return Math.round(value).toLocaleString('en-US'); +} + +/** A duration: `820ms`, `1.4s`, `2m 05s`. */ +export function fmtDuration(ms: number | null | undefined): string { + if (ms === null || ms === undefined || !Number.isFinite(ms)) return '-'; + if (ms < 1000) return `${Math.round(ms)}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + const minutes = Math.floor(ms / 60_000); + const seconds = Math.round((ms % 60_000) / 1000); + return `${minutes}m ${String(seconds).padStart(2, '0')}s`; +} + +/** USD cost: `$0.0012`, `$1.23`; `-` for null. */ +export function fmtCost(usd: number | null | undefined): string { + if (usd === null || usd === undefined || !Number.isFinite(usd)) return '-'; + if (usd === 0) return '$0'; + return usd < 0.01 ? `$${usd.toFixed(4)}` : `$${usd.toFixed(2)}`; +} + +/** Compact `Feb 12, 12:04` for table cells. */ +export function fmtDateTime(epochMs: number | null | undefined): string { + if (epochMs === null || epochMs === undefined) return '-'; + const d = new Date(epochMs); + return `${d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' })}, ${d.toTimeString().slice(0, 5)}`; +} + +/** `3s ago` / `4m ago` / `2h ago` / `1d ago` for the fleet heartbeat column. */ +export function fmtAge(ageMs: number | null | undefined): string { + if (ageMs === null || ageMs === undefined) return 'never'; + if (ageMs < 60_000) return `${Math.max(1, Math.round(ageMs / 1000))}s ago`; + if (ageMs < 3_600_000) return `${Math.round(ageMs / 60_000)}m ago`; + if (ageMs < 86_400_000) return `${Math.round(ageMs / 3_600_000)}h ago`; + return `${Math.round(ageMs / 86_400_000)}d ago`; +} + +/** `12.5%` / `0%`; `-` for null. */ +export function fmtPercent(fraction: number | null | undefined): string { + if (fraction === null || fraction === undefined || !Number.isFinite(fraction)) return '-'; + return `${(fraction * 100).toFixed(fraction > 0 && fraction < 0.01 ? 2 : 1)}%`; +} diff --git a/apps/agent-forge/src/observe/observe.css b/apps/agent-forge/src/observe/observe.css new file mode 100644 index 00000000..92a30a2d --- /dev/null +++ b/apps/agent-forge/src/observe/observe.css @@ -0,0 +1,395 @@ +/* + * Observe view (Traces / Insights / Monitoring / Scores) - styles for the + * panels reading the observe daemon. Reuses layout.css's primitives (.btn, + * .select, .status-pill, .trace-row/.trace-bar waterfall, .agent-card) and + * only adds what the panels themselves need. + */ + +.observe { + display: flex; + flex-direction: column; + min-height: 0; + min-width: 0; + background: var(--bg); +} + +/* The same tab chrome as the bottom drawer. */ +.observe-tabs { + display: flex; + align-items: center; + border-bottom: 1px solid var(--border); + background: var(--surface); + flex: none; + padding: 0 10px; +} +.observe-tab { + padding: 9px 14px; + font-size: 12px; + font-weight: 600; + color: var(--text-muted); + cursor: pointer; + border-bottom: 2px solid transparent; + background: none; + border-left: none; + border-right: none; + border-top: none; +} +.observe-tab.active { + color: var(--text); + border-bottom-color: var(--accent); +} +.observe-tab:focus-visible { + outline: 2px solid var(--accent); + outline-offset: -2px; +} + +.observe-body { + flex: 1; + min-height: 0; + overflow-y: auto; +} +.observe-panel { + padding: 12px 16px; + max-width: 1200px; +} + +.observe-filter-bar { + display: flex; + align-items: flex-end; + gap: 12px; + flex-wrap: wrap; + margin-bottom: 12px; +} +.observe-filter-bar label { + display: flex; + flex-direction: column; + gap: 4px; + font-size: 11px; + font-weight: 600; + color: var(--text-muted); +} +.observe-filter-bar .select { + width: auto; + min-width: 150px; +} + +.observe-table-wrap { + border: 1px solid var(--border); + border-radius: 9px; + overflow: hidden; + background: var(--surface); +} +.observe-table { + width: 100%; + border-collapse: collapse; + font-size: 12.5px; +} +.observe-table th { + text-align: left; + color: var(--text-faint); + font-weight: 600; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.4px; + padding: 7px 12px; + border-bottom: 1px solid var(--border); + background: var(--surface-2); +} +.observe-table td { + color: var(--text); + padding: 6px 12px; + border-bottom: 1px solid var(--border); + vertical-align: middle; +} +.observe-table tbody tr:last-child td { + border-bottom: none; +} +.observe-table.clickable tbody tr { + cursor: pointer; +} +.observe-table tbody tr:hover { + background: var(--surface-2); +} +.observe-num { + text-align: right; + font-family: var(--font-mono); + font-size: 11.5px; +} +.observe-cell-name { + font-weight: 600; + max-width: 260px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.observe-cell-muted { + color: var(--text-muted); + max-width: 320px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.observe-id { + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-muted); +} +.observe-meta { + color: var(--text-faint); + font-size: 11.5px; +} + +.observe-empty { + max-width: 560px; + margin: 48px auto; + padding: 20px; + text-align: center; + color: var(--text-muted); + font-size: 13px; + border: 1px dashed var(--border); + border-radius: 10px; + background: var(--surface); +} +.observe-empty-title { + font-weight: 700; + color: var(--text); + margin-top: 0; +} +.observe-empty code { + font-family: var(--font-mono); + background: var(--surface-2); + border: 1px solid var(--border); + padding: 1px 5px; + border-radius: 5px; + color: var(--text); +} +.observe-loading { + padding: 24px; + color: var(--text-faint); +} +.observe-error { + margin: 8px 0; + padding: 8px 12px; + border: 1px solid var(--danger); + border-radius: 8px; + background: var(--danger-soft); + color: var(--danger); + font-size: 12px; +} +.observe-pager { + display: flex; + align-items: center; + gap: 12px; + margin-top: 10px; + color: var(--text-muted); + font-size: 12px; +} +.observe-pager span { + flex: 1; + text-align: center; +} +.observe-panel-head { + display: flex; + align-items: center; + gap: 10px; + padding: 10px 16px; + border-bottom: 1px solid var(--border); + background: var(--surface); + flex: none; +} + +/* Statuses the daemon reports that the run-status pill set didn't cover yet. */ +.status-pill.status-ok, +.status-pill.status-healthy { + background: var(--success-soft); + color: var(--success); +} +.status-pill.status-stale, +.status-pill.status-interrupted { + background: var(--warning-soft); + color: var(--warning); +} +.status-pill.status-down { + background: var(--danger-soft); + color: var(--danger); +} + +/* ---------- Trace detail: observation waterfall + inspector ---------- */ +.obs-detail { + display: flex; + flex-direction: column; + height: 100%; + min-height: 0; +} +.obs-detail-body { + flex: 1; + min-height: 0; + display: flex; +} +.obs-waterfall { + flex: 1; + min-width: 0; + overflow-y: auto; + padding: 8px; +} +.obs-row .trace-name { + display: flex; + align-items: center; + gap: 6px; + min-width: 0; +} +.obs-kind-dot { + width: 8px; + height: 8px; + border-radius: 3px; + flex: none; +} +.obs-inspector { + width: 340px; + flex: none; + border-left: 1px solid var(--border); + background: var(--surface); + overflow-y: auto; + padding: 12px; +} +.obs-inspector-head { + display: flex; + align-items: center; + gap: 8px; + font-size: 13px; + margin-bottom: 10px; +} +.obs-inspector-head b { + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; +} +.obs-kind { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 700; + text-transform: uppercase; +} +.obs-field { + margin-bottom: 12px; +} +.obs-field-label { + font-size: 10.5px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-faint); + margin-bottom: 4px; +} +.obs-field-value { + font-size: 12px; + color: var(--text); +} +.obs-io { + margin: 0; + padding: 8px; + background: var(--surface-2); + border: 1px solid var(--border); + border-radius: 7px; + font-family: var(--font-mono); + font-size: 11px; + white-space: pre-wrap; + word-break: break-word; + max-height: 220px; + overflow-y: auto; +} +.obs-scores { + margin-top: 14px; +} + +/* ---------- Insights ---------- */ +.insight-cards { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + gap: 10px; + margin-bottom: 14px; +} +.insight-card { + cursor: default; + margin-bottom: 0; +} +.insight-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 8px; + margin-top: 8px; +} +.insight-stat { + display: flex; + flex-direction: column; +} +.insight-value { + font-weight: 700; + font-size: 14px; + font-family: var(--font-mono); +} +.insight-bad { + color: var(--danger); +} +.insight-label { + font-size: 10px; + color: var(--text-faint); + text-transform: uppercase; + letter-spacing: 0.4px; +} +.insight-chart { + display: flex; + align-items: flex-end; + gap: 2px; + height: 120px; + padding: 8px; + border: 1px solid var(--border); + border-radius: 9px; + background: var(--surface); +} +.insight-bar { + flex: 1; + min-width: 3px; + background: var(--accent); + border-radius: 2px 2px 0 0; +} + +/* ---------- Scores ---------- */ +.score-trends { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 10px; + margin-bottom: 14px; +} +.score-trend { + border: 1px solid var(--border); + border-radius: 9px; + background: var(--surface); + padding: 10px 12px; +} +.score-trend-head { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 8px; + margin-bottom: 8px; + font-size: 12.5px; +} +.score-trend-strip { + display: flex; + align-items: flex-end; + gap: 2px; + height: 36px; +} +.score-trend-day { + flex: 1; + min-width: 4px; + border-radius: 2px 2px 0 0; +} +.score-trend-day.pass { + background: var(--success); +} +.score-trend-day.fail { + background: var(--danger); +} diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts new file mode 100644 index 00000000..3c8c3b81 --- /dev/null +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -0,0 +1,175 @@ +/** + * Browser-side client for the observe daemon's read API (`src/observe/routes/*`). + * + * The daemon answers on loopback only and sends no CORS headers, so the + * default base is the same-origin `/observe` proxy: Vite's dev server forwards + * it to the studio API server, which forwards it to the daemon + * (`LOUSHO_OBSERVE_URL`, default `http://127.0.0.1:7777`). An absolute base - + * `VITE_LOUSHO_OBSERVE_URL` at build time, or `localStorage['lousho.observe.url']` + * at runtime - points the client straight at a collector (the hosted path the + * spec leaves open); CORS is then the collector's job. + */ +import { STUDIO_TOKEN_HEADER, loadStudioToken } from '../runtime/studioToken'; +import type { + ObserveAgentRow, + ObserveHealth, + ObserveInsights, + ObserveScoreRow, + ObserveTraceDetail, + ObserveTraceList, +} from './types'; + +const LOCAL_STORAGE_KEY = 'lousho.observe.url'; +const DEFAULT_BASE = '/observe'; + +/** The observe base URL: localStorage override > build env > the same-origin proxy. */ +function observeBaseUrl(): string { + try { + const stored = window.localStorage.getItem(LOCAL_STORAGE_KEY); + if (stored && stored.trim()) return stored.trim().replace(/\/+$/, ''); + } catch { + // Storage blocked (privacy mode): fall through to the env/proxy default. + } + const env = import.meta.env.VITE_LOUSHO_OBSERVE_URL as string | undefined; + if (env && env.trim()) return env.trim().replace(/\/+$/, ''); + return DEFAULT_BASE; +} + +/** Thrown when the daemon can't be reached at all (connection refused, or the studio proxy's 502). */ +export class ObserveDaemonDownError extends Error { + override readonly name = 'ObserveDaemonDownError'; + constructor(message = 'The observe daemon is not reachable.') { + super(message); + } +} + +/** A non-2xx answer from the daemon (400/404/...). */ +export class ObserveApiError extends Error { + override readonly name = 'ObserveApiError'; + constructor( + message: string, + public readonly status: number + ) { + super(message); + } +} + +export interface ObserveClientOptions { + baseUrl?: string; + fetchFn?: typeof fetch; +} + +export interface TraceListFilter { + agent_id?: string; + project?: string; + status?: string; + session_id?: string; + since?: number; + until?: number; + limit?: number; + offset?: number; +} + +export interface ScoreListFilter { + trace_id?: string; + name?: string; + agent_id?: string; + limit?: number; +} + +export interface InsightsFilter { + days?: number; + since?: number; + until?: number; + agent_id?: string; + granularity?: 'hour' | 'day'; +} + +export class ObserveClient { + private readonly baseUrl: string; + private readonly fetchFn: typeof fetch; + + constructor(options: ObserveClientOptions = {}) { + this.baseUrl = options.baseUrl ?? observeBaseUrl(); + // Resolved at call time, not construction: an early `fetch.bind(...)` + // would freeze the pre-test-stub global (and an unbound `fetch` throws + // "Illegal invocation" in some browsers). + this.fetchFn = options.fetchFn ?? ((...args: Parameters) => fetch(...args)); + } + + private url(path: string, query?: Record): string { + const params = new URLSearchParams(); + for (const [key, value] of Object.entries(query ?? {})) { + if (value !== undefined && value !== '') params.set(key, String(value)); + } + const qs = params.toString(); + return `${this.baseUrl}${path}${qs ? `?${qs}` : ''}`; + } + + private async request(path: string, query?: Record): Promise { + const headers = new Headers({ accept: 'application/json' }); + // The studio token only travels same-origin (to the studio server's + // proxy, which requires it); a configured remote collector gets no header. + if (this.baseUrl.startsWith('/')) { + const token = loadStudioToken(); + if (token) headers.set(STUDIO_TOKEN_HEADER, token); + } + let res: Response; + try { + res = await this.fetchFn(this.url(path, query), { headers }); + } catch { + throw new ObserveDaemonDownError(); + } + if (res.status === 502) throw new ObserveDaemonDownError(); + if (!res.ok) { + const body = (await res.json().catch(() => undefined)) as { error?: string } | undefined; + throw new ObserveApiError(body?.error || `Request to ${path} failed with ${res.status}`, res.status); + } + return (await res.json()) as T; + } + + /** `GET /health` - `undefined` when nothing (or not a lousho-observe daemon) answers. */ + async health(): Promise { + try { + const health = await this.request('/health'); + return health.service === 'lousho-observe' && health.ok ? health : undefined; + } catch (error) { + if (error instanceof ObserveDaemonDownError || error instanceof ObserveApiError) return undefined; + throw error; + } + } + + /** `GET /api/v1/traces` - the paginated, filterable trace list. */ + listTraces(filter: TraceListFilter = {}): Promise { + return this.request('/api/v1/traces', { ...filter }); + } + + /** `GET /api/v1/traces/:id` - flat observation rows + scores; `undefined` on the daemon's 404. */ + async traceDetail(traceId: string): Promise { + try { + return await this.request(`/api/v1/traces/${encodeURIComponent(traceId)}`); + } catch (error) { + if (error instanceof ObserveApiError && error.status === 404) return undefined; + throw error; + } + } + + /** `GET /api/v1/agents` - the fleet registry with read-time freshness. */ + async listAgents(project?: string): Promise { + const body = await this.request<{ agents: ObserveAgentRow[] }>('/api/v1/agents', { project }); + return body.agents; + } + + /** `GET /api/v1/insights` - per-agent cards + rollup series. */ + insights(filter: InsightsFilter = {}): Promise { + return this.request('/api/v1/insights', { ...filter }); + } + + /** `GET /api/v1/scores` - eval/judge/human score rows. */ + async listScores(filter: ScoreListFilter = {}): Promise { + const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { ...filter }); + return body.scores; + } +} + +export const observeClient = new ObserveClient(); diff --git a/apps/agent-forge/src/observe/tree.ts b/apps/agent-forge/src/observe/tree.ts new file mode 100644 index 00000000..ff73d299 --- /dev/null +++ b/apps/agent-forge/src/observe/tree.ts @@ -0,0 +1,88 @@ +/** + * The daemon returns a trace's observations as flat rows ordered by `start` + * (`GET /api/v1/traces/:id`); the panels nest them by `parent_obs_id` + * client-side. Rows whose parent is missing from the set (a dropped batch, + * a parent written after its children) are hoisted to root level rather + * than dropped - a partial tree beats a silently empty one. + */ +import type { ObserveObservationRow } from './types'; + +export interface ObservationNode { + row: ObserveObservationRow; + children: ObservationNode[]; + /** Tree depth, roots = 0 - the waterfall indents by it. */ + depth: number; +} + +function byStart(a: ObservationNode, b: ObservationNode): number { + return a.row.start - b.row.start; +} + +/** + * Flat rows -> forest of {@link ObservationNode}s, children sorted by `start`. + * Parent ids that don't resolve (and parent/self cycles) land at the roots. + */ +export function nestObservations(rows: ObserveObservationRow[]): ObservationNode[] { + const ids = new Set(rows.map((row) => row.obs_id)); + const nodes = new Map(); + for (const row of rows) nodes.set(row.obs_id, { row, children: [], depth: 0 }); + + const roots: ObservationNode[] = []; + for (const node of nodes.values()) { + const parentId = node.row.parent_obs_id; + const parent = parentId !== null && ids.has(parentId) ? nodes.get(parentId) : undefined; + if (parent === undefined || parent === node) roots.push(node); + else parent.children.push(node); + } + + // Depth-first pass assigns depth and sorts siblings; a parent_obs_id cycle + // that never reaches a root is broken open at its earliest-started member. + const seen = new Set(); + const visit = (node: ObservationNode, depth: number) => { + if (seen.has(node.row.obs_id)) return; + seen.add(node.row.obs_id); + node.depth = depth; + node.children.sort(byStart); + for (const child of node.children) visit(child, depth + 1); + }; + roots.sort(byStart); + for (const root of roots) visit(root, 0); + // Cycle leftovers: reattach the cycle's earliest member at root level. + const leftovers = rows.filter((row) => !seen.has(row.obs_id)).sort((a, b) => a.start - b.start); + for (const row of leftovers) { + const node = nodes.get(row.obs_id); + if (node && !seen.has(node.row.obs_id)) { + roots.push(node); + visit(node, 0); + } + } + return roots; +} + +/** Depth-first flattening for the waterfall: each node once, in display order. */ +export function flattenObservationTree(roots: ObservationNode[]): ObservationNode[] { + const flat: ObservationNode[] = []; + const seen = new Set(); + const visit = (node: ObservationNode) => { + // A parent_obs_id cycle nests the cycle members under each other; emit + // each obs_id once rather than recursing through it forever. + if (seen.has(node.row.obs_id)) return; + seen.add(node.row.obs_id); + flat.push(node); + for (const child of node.children) visit(child); + }; + for (const root of roots) visit(root); + return flat; +} + +/** `[start, end]` bounds over every observation (an unfinished row counts to `now`). */ +export function traceBounds(rows: ObserveObservationRow[], now = Date.now()): { minStart: number; maxEnd: number } { + if (rows.length === 0) return { minStart: 0, maxEnd: 1 }; + let minStart = Infinity; + let maxEnd = -Infinity; + for (const row of rows) { + minStart = Math.min(minStart, row.start); + maxEnd = Math.max(maxEnd, row.end ?? now); + } + return { minStart, maxEnd: Math.max(maxEnd, minStart + 1) }; +} diff --git a/apps/agent-forge/src/observe/types.ts b/apps/agent-forge/src/observe/types.ts new file mode 100644 index 00000000..381e280f --- /dev/null +++ b/apps/agent-forge/src/observe/types.ts @@ -0,0 +1,128 @@ +/** + * The observe daemon's wire shapes, mirrored from `src/observe/store.ts`'s + * read models (TraceRow, ObservationRow, ScoreRow, AgentRow, AgentInsights, + * RollupRow). Kept as local declarations rather than importing the SDK's + * internals: the panels' contract is the daemon's HTTP JSON, not its TS + * modules - the same contract a hosted collector implements later. + */ + +/** `GET /health` - the `service` marker is how the panels tell "the daemon" apart from whatever else grabbed the port. */ +export interface ObserveHealth { + service: string; + ok: boolean; + version?: string; +} + +/** One row of `GET /api/v1/traces`: an aggregated trace header (status comes from the root observation). */ +export interface ObserveTraceRow { + trace_id: string; + name: string | null; + agent_id: string | null; + session_id: string | null; + start: number; + end: number; + status: string; + observations: number; + generations: number; + tools: number; + errors: number; + input_tokens: number; + output_tokens: number; + cost_usd: number; +} + +export interface ObserveTraceList { + traces: ObserveTraceRow[]; + total: number; + limit: number; + offset: number; +} + +/** One flat observation row of `GET /api/v1/traces/:id` - the client nests them by `parent_obs_id`. */ +export interface ObserveObservationRow { + obs_id: string; + trace_id: string; + session_id: string | null; + agent_id: string | null; + parent_obs_id: string | null; + kind: string; + name: string | null; + start: number; + end: number | null; + status: string | null; + input: string | null; + output: string | null; + model: string | null; + usage_input_tokens: number | null; + usage_output_tokens: number | null; + usage_cached_tokens: number | null; + usage_reasoning_tokens: number | null; + cost_usd: number | null; + cost_partial: boolean | null; + usage_estimated: boolean | null; + truncated: boolean; + attrs: Record | null; +} + +export interface ObserveScoreRow { + score_id: string; + trace_id: string; + obs_id: string | null; + name: string; + value: number | null; + source: string; + comment: string | null; + agent_id: string | null; + created_at: number; +} + +export interface ObserveTraceDetail { + trace_id: string; + observations: ObserveObservationRow[]; + scores: ObserveScoreRow[]; +} + +/** `GET /api/v1/agents` row - `status`/`heartbeat_age_ms` are the daemon's read-time freshness derivation. */ +export interface ObserveAgentRow { + agent_id: string; + project: string | null; + name: string | null; + version: string | null; + first_seen: number; + last_heartbeat: number | null; + status: string; + heartbeat_age_ms: number | null; + heartbeat_interval_ms: number | null; + hostname: string | null; + pid: number | null; +} + +export interface ObserveAgentInsights { + agent_id: string; + project: string | null; + name: string | null; + runs: number; + runs_per_day: number; + errors: number; + error_rate: number; + cost_usd: number; + input_tokens: number; + output_tokens: number; + p50_latency_ms: number | null; + p95_latency_ms: number | null; +} + +export interface ObserveRollupRow { + bucket_start: number; + granularity: string; + agent_id: string; + metric: string; + value: number; +} + +/** `GET /api/v1/insights`. */ +export interface ObserveInsights { + window: { since: number; until: number }; + agents: ObserveAgentInsights[]; + series: ObserveRollupRow[]; +} diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts new file mode 100644 index 00000000..e331a15a --- /dev/null +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -0,0 +1,102 @@ +/** + * React hooks over {@link ObserveClient}. `useObserveQuery` is the shared + * fetch/cancel/refresh plumbing; the per-endpoint hooks below map 1:1 onto + * the daemon's read routes. Every hook re-fetches when its `refreshKey` + * argument changes, so panels can wire a Refresh button (or a poll) without + * duplicating effect logic. + */ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { observeClient, ObserveDaemonDownError, type InsightsFilter, type ScoreListFilter, type TraceListFilter } from './observeClient'; +import type { + ObserveAgentRow, + ObserveHealth, + ObserveInsights, + ObserveScoreRow, + ObserveTraceDetail, + ObserveTraceList, +} from './types'; + +export interface ObserveQuery { + data: T | undefined; + /** The fetch failure, if any - `ObserveDaemonDownError` means "run `lousho observe start`". */ + error: Error | undefined; + /** True while the first fetch is in flight (refreshes keep the old data and don't flag it). */ + loading: boolean; + /** Re-runs the fetch. */ + refresh: () => void; +} + +function useObserveQuery(fetcher: () => Promise, deps: readonly unknown[]): ObserveQuery { + const [data, setData] = useState(undefined); + const [error, setError] = useState(undefined); + const [loading, setLoading] = useState(true); + const [tick, setTick] = useState(0); + // The latest fetcher's identity isn't a dep - callers pass `deps` for that; + // keeping it in a ref avoids an unstable-inline-callback refetch loop. + const fetcherRef = useRef(fetcher); + fetcherRef.current = fetcher; + + useEffect(() => { + let cancelled = false; + setLoading(true); + fetcherRef + .current() + .then((result) => { + if (cancelled) return; + setData(result); + setError(undefined); + }) + .catch((e: unknown) => { + if (cancelled) return; + setData(undefined); + setError(e instanceof Error ? e : new Error(String(e))); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + return () => { + cancelled = true; + }; + // Callers own `deps` (fetcher identity is a ref, not a dep). + }, [...deps, tick]); + + const refresh = useCallback(() => setTick((t) => t + 1), []); + return { data, error, loading, refresh }; +} + +/** `GET /health`, re-probed on `pollMs` (default 5s) so the panels wake up once `lousho observe start` runs. */ +export function useDaemonHealth(pollMs = 5000): ObserveQuery & { down: boolean } { + const query = useObserveQuery(() => observeClient.health(), []); + const down = query.error instanceof ObserveDaemonDownError || (query.data === undefined && !query.loading); + useEffect(() => { + if (pollMs <= 0) return undefined; + const timer = setInterval(query.refresh, pollMs); + return () => clearInterval(timer); + // `query.refresh` is a stable useCallback; re-arming only on pollMs. + }, [pollMs, query.refresh]); + return { ...query, down }; +} + +export function useTraces(filter: TraceListFilter, refreshKey?: unknown): ObserveQuery { + const query = useObserveQuery(() => observeClient.listTraces(filter), [JSON.stringify(filter), refreshKey]); + return query; +} + +export function useTraceDetail(traceId: string | undefined, refreshKey?: unknown): ObserveQuery { + return useObserveQuery( + () => (traceId === undefined ? Promise.resolve(undefined) : observeClient.traceDetail(traceId)), + [traceId, refreshKey] + ); +} + +export function useAgents(project?: string, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listAgents(project), [project, refreshKey]); +} + +export function useInsights(filter: InsightsFilter, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.insights(filter), [JSON.stringify(filter), refreshKey]); +} + +export function useScores(filter: ScoreListFilter, refreshKey?: unknown): ObserveQuery { + return useObserveQuery(() => observeClient.listScores(filter), [JSON.stringify(filter), refreshKey]); +} diff --git a/apps/agent-forge/src/state/AppState.tsx b/apps/agent-forge/src/state/AppState.tsx index 71ebaf2e..f80eadce 100644 --- a/apps/agent-forge/src/state/AppState.tsx +++ b/apps/agent-forge/src/state/AppState.tsx @@ -22,6 +22,8 @@ import { useActiveProfile, useChatControls, useRunControls } from './useAgentCon type RailTab = 'agents' | 'nodes'; export type DrawerTab = 'chat' | 'logs' | 'trace' | 'output' | 'history' | 'settings'; +/** Top-level surface: the builder canvas, or the observability panels (Traces/Insights/Monitoring/Scores). */ +type StudioView = 'build' | 'observe'; interface AppState { store: AgentStore; @@ -53,6 +55,9 @@ interface AppState { setRailTab: (tab: RailTab) => void; drawerTab: DrawerTab; setDrawerTab: (tab: DrawerTab) => void; + /** Which surface the middle of the app shows - the agent builder canvas or the observe panels. */ + view: StudioView; + setView: (view: StudioView) => void; /** Every agent currently known to `store`, refreshed after save/create/switch. */ agents: AgentStoreEntry[]; /** Loads a different agent's graph into the canvas - a full state swap, not a visual pan. */ @@ -150,6 +155,7 @@ export function AppStateProvider({ children }: { children: ReactNode }) { const store = useMemo(() => new HttpAgentStore(runtimeClient), []); const [railTab, setRailTab] = useState('agents'); const [drawerTab, setDrawerTab] = useState('chat'); + const [view, setView] = useState('build'); const doc = useAgentDocument(store); const { agentId, spec } = doc; const agentStatuses = useAgentStatuses(doc.agents); @@ -165,6 +171,8 @@ export function AppStateProvider({ children }: { children: ReactNode }) { setRailTab, drawerTab, setDrawerTab, + view, + setView, ...run, agentStatuses, ...stream, diff --git a/apps/agent-forge/vite.config.ts b/apps/agent-forge/vite.config.ts index bdf96bf1..fd64dccd 100644 --- a/apps/agent-forge/vite.config.ts +++ b/apps/agent-forge/vite.config.ts @@ -22,6 +22,10 @@ export default defineConfig({ }, // Time-travel routes (history, fork, compare). '/runs': { target: process.env.LOUSHO_STUDIO_API_URL ?? 'http://127.0.0.1:4750', changeOrigin: true }, + // The observe panels' read path - proxied through the studio API + // server (same as prod; it forwards to the `lousho observe` daemon, + // which serves no CORS headers of its own). + '/observe': { target: process.env.LOUSHO_STUDIO_API_URL ?? 'http://127.0.0.1:4750', changeOrigin: true }, }, }, test: { diff --git a/bin/lousho.js b/bin/lousho.js index d23b4b24..12befd8f 100755 --- a/bin/lousho.js +++ b/bin/lousho.js @@ -33,6 +33,10 @@ const USAGE = [ ' Run *.eval.ts files under your vitest and report the results.', ' lousho traces [traceId|prefix] [--dir D] [--limit N] [--json] [--content]', ' List saved traces (fileTraceExporter) or print one as a span tree.', + ' lousho observe [--port N] [--db path] [--host H] [--require-token] [--project X]', + ' Run the observability collector agents emit to when LOUSHO_OBSERVE=1.', + ' lousho prompt [args] [--url U | --port N] [--token T] [--json]', + ' Manage the prompt registry the collector serves: push a file as a new version, list, or get name[@version].', ' lousho --version', ' Print the package version.', '', @@ -74,6 +78,10 @@ const COMMANDS = new Map([ ['eval', { module: 'eval', run: 'runEval', env: true }], // Lists the runs fileTraceExporter() saved, or prints one as a span tree; see src/cli/traces.ts. ['traces', { module: 'traces', run: 'runTraces' }], + // The observability collector daemon (src/observe); see src/cli/observe.ts. + ['observe', { module: 'observe', run: 'runObserve' }], + // The prompt registry client of the observe daemon (src/observe); see src/cli/prompt.ts. + ['prompt', { module: 'prompt', run: 'runPrompt' }], ]); const BYTE_ORDER_MARK = 0xfeff; diff --git a/docs/cli.md b/docs/cli.md index ae94b580..165be36e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -36,6 +36,8 @@ lousho mcp [--http --port N --host H] lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping] lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--config vitest.config.ts] [--timeout ] lousho traces [traceId|prefix] [--dir D] [--limit N] [--json] [--content] +lousho observe [--port N] [--db path] [--host H] [--require-token] [--project X] [--webhook URL] +lousho prompt |list|get > [--url U] [--port N] [--token T] [--json] [--name N] lousho --version ``` @@ -265,3 +267,32 @@ npx lousho studio --port 5000 Starts one local server for the Agent Forge API and UI. Agents and run state are stored under `.lousho/` in the current directory. See [Agent Forge](./agent-forge.md) for the walkthrough. + +## `lousho observe` + +```bash +npx lousho observe start # collector on http://127.0.0.1:7777 +npx lousho observe status # health + fleet counts +npx lousho observe stop # pidfile + SIGTERM +npx lousho observe rebuild-rollups # recompute insight aggregates +npx lousho observe token create --project myapp # bearer key for non-loopback auth +``` + +Runs the local collector every agent reports to when `LOUSHO_OBSERVE` is +set: traces, fleet monitoring, eval scores, the prompt registry and the +context hub in one SQLite file (default `~/.lousho/observe.db`). The Studio +Observe tabs read it. `start` takes `--port` (or `LOUSHO_OBSERVE_PORT`), +`--db`, `--host`, `--require-token` and a repeatable `--webhook` for alert +hooks. See [The observe daemon](./observability.md#the-observe-daemon). + +## `lousho prompt` + +```bash +npx lousho prompt push prompt.md --name mira # publishes a new version +npx lousho prompt list +npx lousho prompt get mira@latest +``` + +Publishes and reads prompt versions in the observe daemon's registry. Agents +resolve them with `createAgent({ prompt: 'mira@latest' })` or +`LOUSHO_PROMPT`. See [The observe daemon](./observability.md#the-observe-daemon). diff --git a/docs/errors.md b/docs/errors.md index a88f904f..4e7a4f15 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -96,7 +96,7 @@ Find a code by area: | [Schedules](#schedules) | [`LOUSHO_SCHEDULE_INVALID`](#lousho_schedule_invalid), [`LOUSHO_SCHEDULE_RUN_INCOMPLETE`](#lousho_schedule_run_incomplete) | Defining or loading a schedule, or a scheduled turn that did not finish. | | [Channels](#channels) | [`LOUSHO_CHANNEL_INVALID`](#lousho_channel_invalid), [`LOUSHO_MEMORY_INVALID`](#lousho_memory_invalid) | Defining a channel or a memory slot. | | [Registry](#registry) | [`LOUSHO_REGISTRY_UNREACHABLE`](#lousho_registry_unreachable), [`LOUSHO_REGISTRY_ITEM_NOT_FOUND`](#lousho_registry_item_not_found), [`LOUSHO_REGISTRY_INVALID`](#lousho_registry_invalid), [`LOUSHO_REGISTRY_UNSAFE_PATH`](#lousho_registry_unsafe_path), [`LOUSHO_REGISTRY_FILE_EXISTS`](#lousho_registry_file_exists), [`LOUSHO_REGISTRY_MANIFEST_MISMATCH`](#lousho_registry_manifest_mismatch) | `lousho add` fetching or copying from a registry. | -| [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied) | Asking a sandbox for something it cannot do on the current platform, or to write outside its root. | +| [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied), [`LOUSHO_SANDBOX_FAILED`](#lousho_sandbox_failed) | Asking a sandbox for something it cannot do on the current platform, writing outside its root, or a code-sandbox boundary (timeout / OOM / driver) failing a run. | | [Agent directories, skills and flows](#agent-directories-skills-and-flows) | [`LOUSHO_AGENT_DIR_INVALID`](#lousho_agent_dir_invalid), [`LOUSHO_SKILL_INVALID`](#lousho_skill_invalid), [`LOUSHO_FLOW_INVALID`](#lousho_flow_invalid), [`LOUSHO_FLOW_TOOL_DENIED`](#lousho_flow_tool_denied), [`LOUSHO_FLOW_CHANGED`](#lousho_flow_changed) | Loading an agent directory, a skill, or a flow definition, and gating a flow's tool calls. | | [Storage, deployment and integrations](#storage-deployment-and-integrations) | [`LOUSHO_STORAGE_FAILED`](#lousho_storage_failed), [`LOUSHO_STORAGE_BUSY`](#lousho_storage_busy), [`LOUSHO_TRIGGER_INVALID`](#lousho_trigger_invalid), [`LOUSHO_CHANNEL_REQUEST_FAILED`](#lousho_channel_request_failed), [`LOUSHO_DEPLOY_FAILED`](#lousho_deploy_failed), [`LOUSHO_SERVER_UNPROTECTED`](#lousho_server_unprotected) | A storage backend, a trigger, a channel request, `lousho build` or the built server's start-up. | | [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. | @@ -962,6 +962,23 @@ directory at construction). See [Guardrails](./guardrails.md#sandboxed-tools). **Example:** `sandbox.writeFile('/etc/profile.d/x.sh', ...)` with the default root. +### LOUSHO_SANDBOX_FAILED + +**Means:** a `sandbox()` / `defineTool({ sandbox })` run failed at the +sandbox boundary - not in the tool's own code, whose errors cross back +unchanged. The `SandboxError`'s `kind` names which boundary: `'timeout'` +(the run outlasted `timeoutMs`), `'oom'` (the driver killed it for its +memory cap - with the `'isolate'` driver this can also take the host +process, since the isolate is an execution boundary, not a security one) +or `'driver'` (the driver itself failed). The tool span carries the same +kind as `sandbox.error`. + +**Fix:** raise `memoryMb`/`timeoutMs`, or fix the driver; for a missing +`isolated-vm` install the optional peer or pass another `SandboxDriver`. + +**Example:** `sandbox({ driver: 'isolate', timeoutMs: 5_000 })` running code +that does not return in five seconds. + ## Agent directories, skills and flows ### LOUSHO_AGENT_DIR_INVALID diff --git a/docs/observability.md b/docs/observability.md index 03f5cb56..1411342b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -379,6 +379,39 @@ and prompt and tool content only if the agent sets `redactContent: false` or `captureContent: true`, so nothing is written to disk unless you ask. The spec file format has no traces setting. +## The observe daemon + +`lousho observe start` runs a local collector on `http://127.0.0.1:7777` +(loopback only, no auth needed). When `LOUSHO_OBSERVE` is set, every agent +built on this SDK reports spans to it automatically — `1` (or `true`) emits +to the default URL, a URL value emits there. `LOUSHO_OBSERVE_PROJECT` labels +the rows; `LOUSHO_OBSERVE_TOKEN` sends a bearer token for daemons bound +beyond loopback (`--require-token` plus `lousho observe token create`). + +One SQLite file (default `~/.lousho/observe.db`, WAL mode) holds the fleet's +observations, sessions, eval scores, prompt versions and shared context. +`npx lousho studio` renders it in the Observe tabs: trace trees with latency +waterfalls, per-agent insights (runs/day, p50/p95 latency, cost, error +rate), a live fleet monitor (heartbeat-derived status — a crashed agent's +open sessions flip to `interrupted`), and eval score history. Alerts on +`down` transitions or error-rate thresholds POST to `--webhook` URLs or an +`alerts.json` next to the db. + +The daemon also serves two stores agents can use at runtime: + +- **Prompt registry.** `createAgent({ prompt: 'name@version' })` or + `LOUSHO_PROMPT` fetches instructions from it at build (`@latest` resolves + on the daemon; an unreachable daemon falls back to `instructions`). Every + run logs the exact version used. `lousho prompt push` publishes a new + version. +- **Context hub.** `hub()` / `hubTools()` let agents publish and query + namespaced key-values scoped `private` (the writing agent only) or + `fleet` (every agent on the daemon). + +Reporting is fire-and-forget: a down daemon drops and counts events, it +never blocks a run. Eval scores flow through the same pipe — +`lousho eval --observe` emits them even when `LOUSHO_OBSERVE` is unset. + ## The built server The server `lousho build --target=node-server` or `--target=docker` generates diff --git a/docs/specs/0001-observability-platform/0001-evaluators.md b/docs/specs/0001-observability-platform/0001-evaluators.md new file mode 100644 index 00000000..85a10bdf --- /dev/null +++ b/docs/specs/0001-observability-platform/0001-evaluators.md @@ -0,0 +1,27 @@ +# 0001-evaluators: scores beside runs + +## Summary + +`src/evals` already defines evals, judges, and cassettes; this child makes their results first class platform data. Every eval run emits score rows tied to the trace it evaluated, so regressions are visible in Studio beside the runs that caused them. + +## Decision + +Eval execution emits `score` ingest events: `lousho eval` results, `llmJudge` verdicts, and any agent self eval attach to the run's `trace_id`. Studio shows scores on the trace detail and an eval history view per agent and per eval name. + +## Surface + +**Emitter**: eval runners call the same reporter. Score mapping from the existing `EvalResult` shape: one score row per `AssertionResult`, `name` = `/`, `llmJudge` verdicts land with `source: "judge"`, everything else `source: "eval"`. `trace_id` binds to the *exercised run's* trace; the eval harness run gets its own linked trace so judgement context stays inspectable. + +**Studio eval panels**: +- Scores on trace detail: inline with the observations they judged. +- Eval history: per eval name, score over time, pass rate trend, linked traces for failures. +- CI surface: `lousho eval --observe` ships scores to the daemon, so CI runs appear in the same history. + +## Deferred to its FEATURE spec + +- Whether human feedback scores (a thumbs widget in Studio) write back through ingest or a Studio only route. +- How drift detection (`src/evals/drift.ts`) surfaces: as scores, as a separate panel, or both. + +## Rationale + +Evals without trace linkage are numbers without context: a failing score must reach the run that produced it in one click. Reusing the ingest pipeline keeps eval data inside the same store and retention story instead of a second system (basis: Langfuse scores attach to traces the same way). diff --git a/docs/specs/0001-observability-platform/0001-monitoring.md b/docs/specs/0001-observability-platform/0001-monitoring.md new file mode 100644 index 00000000..cf1b6e7e --- /dev/null +++ b/docs/specs/0001-observability-platform/0001-monitoring.md @@ -0,0 +1,27 @@ +# 0001-monitoring: the fleet registry + +## Summary + +Live view over every agent reporting to the daemon: which bots are up, what they are doing, whether they are erroring. Alert hooks fire when an agent goes quiet or its error rate spikes. + +## Decision + +Agents register on first emit and heartbeat on an interval (default 60s) via the same ingest envelope (`agent.heartbeat` events). The `agents` table is the registry: `status` derives from heartbeat freshness (`healthy` < 2 intervals, `stale`, `down` beyond a threshold), not from explicit health checks. + +## Surface + +**Daemon**: heartbeat events upsert `agents`; a periodic sweep (every 30s) recomputes `status` and fires alert hooks on transitions. A graceful exit sends `agent.deregister` (status `stopped`); when the sweep marks an agent `down` without one (a crash), it also flips that agent's open `sessions` and unfinished `observations` to `interrupted`, so dead runs do not look alive forever. + +**Studio Monitoring panel**: fleet table (name, project, status, last heartbeat, runs today, error rate today from rollups), drill into an agent's recent traces. + +**Alert hooks**: a `alerts` config on the daemon (`lousho observe --webhook ` or an `alerts.json`): on `healthy → down` or error rate crossing a configured threshold, POST a JSON alert to configured webhook URLs (Telegram bot message is the obvious first consumer for the owner's fleet). + +## Deferred to its FEATURE spec + +- Alert threshold values and whether they are per agent or global. +- Whether stale agents auto prune from the registry. +- The alert payload schema consumers depend on. + +## Rationale + +Heartbeat plus freshness derived status avoids a separate health check protocol; the data already flowing tells us liveness for free. Webhook only alerts keep the daemon dependency free (no SMTP, no SDK for chat apps) while still reaching the user's Telegram immediately. diff --git a/docs/specs/0001-observability-platform/0001-prompts-context-hub.md b/docs/specs/0001-observability-platform/0001-prompts-context-hub.md new file mode 100644 index 00000000..79caba51 --- /dev/null +++ b/docs/specs/0001-observability-platform/0001-prompts-context-hub.md @@ -0,0 +1,30 @@ +# 0001-prompts-context-hub: registry and shared memory + +## Summary + +Two related stores in one child. The prompt registry versions agent instructions in the daemon; agents fetch `name@version` at run start and every run logs which version it used. The Context Hub is a shared memory store agents can publish to and query, scoped `private` (one agent) or `fleet` (all agents on the daemon). + +## Decision + +Prompts live in the `prompts` table, edited in Studio or pushed via `lousho prompt push`. An agent opts in with `createAgent({ prompt: 'mira@latest' })` or env `LOUSHO_PROMPT`; the SDK fetches text at agent build, falls back to embedded instructions when the daemon is unreachable. `prompt_uses` rows link every run to the exact prompt version that produced it, closing the loop with evals. + +Context Hub entries are namespaced key values in `hub_entries`; agents get `hub.publish` / `hub.query` tools (or memory API calls) when `LOUSHO_OBSERVE` is set. `scope: private` entries are visible only to the writing `agent_id`; `scope: fleet` to all. + +## Surface + +**Registry API** (on the daemon): `GET /api/v1/prompts/:name/:version` for agents, `POST /api/v1/prompts` for publish, list and diff endpoints for Studio. `@latest` is a movable label resolved daemon side, never in the SDK, so "latest" means the same thing for every agent at once. Each fetch that lands in a run emits a `prompt_use` ingest event (in the shared envelope), which is what writes `prompt_uses` rows. + +**Hub API**: `POST /api/v1/hub` (publish) and `GET /api/v1/hub?namespace&scope` (query). Scope enforcement happens daemon side using the caller's `agent_id`; the SDK just asks. + +**Studio**: prompt editor with version history and a "used by N runs" link into traces; hub browser with namespace filtering. + +## Deferred to its FEATURE spec + +- Prompt variable templating (mustache vs plain interpolation) and validation on publish. +- Fleet scope write policy: who may publish fleet entries (any agent vs an allowlist). +- Hub TTL and size limits per entry. +- Whether hub entries should feed `memory_search` retrieval automatically. + +## Rationale + +Prompt versioning is the single highest leverage observability feature after traces: "what did this run see" is unanswerable today. Fetch at build keeps agents runnable when the daemon is down (embedded instructions are the fallback), which matters more than push convenience. Shared memory between agents is how Mira's learned facts reach Ghost without copy pasting; scope keeps it a deliberate share, not an automatic leak. diff --git a/docs/specs/0001-observability-platform/0001-sandboxing.md b/docs/specs/0001-observability-platform/0001-sandboxing.md new file mode 100644 index 00000000..704a0439 --- /dev/null +++ b/docs/specs/0001-observability-platform/0001-sandboxing.md @@ -0,0 +1,33 @@ +# 0001-sandboxing: isolated tool execution + +## Summary + +A driver interface for running untrusted tool code outside the agent's own process context. v1 ships an `isolated-vm` driver (in process V8 isolate); the seam leaves room for Docker or E2B sidecars later. + +## Decision + +`sandbox({ driver })` is a public API: a driver runs a function or module in an isolated context with controlled globals and a result channel. `defineTool` gains an optional `sandbox` field so a tool can declare it runs isolated. The `isolate` driver wraps `isolated-vm`; it is an optional dependency, so core users never install it. + +## Surface + +```ts +const box = sandbox({ driver: 'isolate', memoryMb: 128, timeoutMs: 5_000 }); +const result = await box.run(userCode, { args: [input] }); +// run(code: string, opts?): Promise — code and args cross the +// boundary by structured clone; the returned value must clone back. +``` + +- `driver`: `'isolate'` in v1; `'docker'` and `'e2b'` reserved. +- `memoryMb`, `timeoutMs`: resource guards enforced by the driver; a timeout or OOM kill maps to an observation with `status: "error"` and `sandbox.error` (`timeout` | `oom` | `driver`) in `attrs`. +- Sandbox runs emit `observations` of kind `tool` with `sandbox.driver` in `attrs`, so isolated execution shows in traces. +- Stated plainly on the API: `isolate` is an *execution* boundary, not a security boundary; an OOM can take the host process. Do not run code you would not run yourself; hostile code needs the future container driver. + +## Deferred to its FEATURE spec + +- The capability model inside the isolate (which globals and modules reach it; default deny list). +- Whether `pc_*` style machine tools can ever route through a sandbox (probably never; they are the host by design). +- Docker and E2B driver contracts. + +## Rationale + +`isolated-vm` is the fast in process isolate and is maintained, though in maintenance mode with known limits (no multi process isolation, OOM can crash the host) (basis: landscape check). That is acceptable for "run this LLM written snippet" cases and unacceptable for truly hostile code, which is exactly why the driver seam matters more than the engine: swapping to containers is a config change, not an API break (basis: TanStack AI isolate driver pattern). diff --git a/docs/specs/0001-observability-platform/0001-tracing-insights.md b/docs/specs/0001-observability-platform/0001-tracing-insights.md new file mode 100644 index 00000000..0c3c1492 --- /dev/null +++ b/docs/specs/0001-observability-platform/0001-tracing-insights.md @@ -0,0 +1,32 @@ +# 0001-tracing-insights: the data plane + +## Summary + +The core of the platform: every agent run becomes observation rows in the collector's SQLite store, queryable as traces in Studio. Insights is aggregate analytics over those rows: latency, cost, error rate, token usage, per agent and per project. + +## Decision + +Build the data plane first; it is the dependency for every other child. The SDK gains an in process reporter that drains run events into batched `POST /api/v1/ingest` calls when `LOUSHO_OBSERVE` is set. The daemon writes them into the `observations` wide table and maintains `rollups` for Insights queries. + +## Surface + +**Reporter** (`src/observe/reporter.ts`): subscribes to the existing run events (the same stream `src/traces` produces), buffers them, flushes on size (default 50 events) or interval (default 5s), flushes on process exit with a ~2s cap. Crash loss is accepted and bounded: nothing persists mid run, and unflushed events count into the same `dropped` metric as daemon unreachable. Failure mode stays drop and count, never block a run. The reporter's own HTTP calls to the daemon are excluded from instrumentation or it recurses. + +**Daemon write path**: ingest validates the envelope, upserts by `obs_id` (resends are idempotent), updates the agent registry `last_heartbeat`, and recomputes affected `rollups` bucket rows inside the same transaction (`INSERT … SELECT … GROUP BY`, so a resent batch can never double count; see the index contract). + +**Studio panels** (new in `apps/agent-forge`): +- Traces list: session and trace rows per project, filterable by status, agent, time. +- Trace detail: the observation tree (spans, generations, tool calls) with inputs, outputs, token usage, and latency waterfall. +- Insights: per agent cards (runs/day, p50/p95 latency, cost, error rate) plus time series from `rollups`. + +**Insights queries** read `rollups` for aggregates and `observations` for drilldown; no query hits raw rows for dashboard level numbers. + +## Deferred to its FEATURE spec + +- Exact rollup granularities and retention (minute rows for 24h, hour rows for 30d is the likely shape). +- Trace list pagination and filtering contract. +- The `/api/v1/query` read endpoints' exact shape (decided at index: HTTP only, daemon owns the db). + +## Rationale + +Traces are the substrate every other surface reads; building them first is the only ordering that avoids rework. The reporter reuses the existing event stream instead of a new instrumentation layer (basis: `src/traces` already emits run and step events). Denormalized rows keep every common query (per trace, per session, per agent, per model) a single table scan (basis: Langfuse v4 observations first model, in `rationale.md`). diff --git a/docs/specs/0001-observability-platform/index.md b/docs/specs/0001-observability-platform/index.md new file mode 100644 index 00000000..582c6ddc --- /dev/null +++ b/docs/specs/0001-observability-platform/index.md @@ -0,0 +1,127 @@ +# 0001. Observability platform for the SDK + +**Date**: 2026-10-22 +**Status**: Proposed + +## Summary + +Build an observability platform inside the SDK so every agent run, step, tool call, and eval score lands in one searchable place. A small local daemon collects data from every agent on the machine into one SQLite file; agents opt in with an env var and need no code changes. The shape is a local version of LangSmith on purpose, so a hosted collector can implement the same ingest API later with no SDK changes. + +## Structure + +Umbrella spec. Each child is one subsystem decision under this platform: + +- [0001-tracing-insights.md](0001-tracing-insights.md): the data plane. Observation rows, the ingest API, Insights rollups, Studio trace panels. The value core, built first. +- [0001-monitoring.md](0001-monitoring.md): the fleet registry. Agent heartbeats, live status, error rates, alert hooks. +- [0001-evaluators.md](0001-evaluators.md): eval scores as first class rows beside runs, plus Studio and CI surfaces over the existing `src/evals`. +- [0001-prompts-context-hub.md](0001-prompts-context-hub.md): the versioned prompt registry and the shared memory hub between agents. +- [0001-sandboxing.md](0001-sandboxing.md): the sandbox driver interface and the isolated-vm driver for untrusted tool code. + +## Decision + +**Chosen option**: Option 2: hybrid platform, local first. + +A local collector daemon (`lousho observe`) owns one SQLite store for the whole machine. SDK agents emit batched observation events to it over an open HTTP ingest API when `LOUSHO_OBSERVE` is set (or `createAgent({ observability })` is passed). The dashboard is new panels inside the existing `apps/agent-forge` Studio app, not a new app. The ingest API is the contract a hosted collector implements later, which is what makes this a LangSmith clone rather than just local logging. + +**Implementation skills**: none installed shaped this design. + +## Proposed stack + +| Layer | Choice | Reason | +|---|---|---| +| Language | TypeScript, ESM | The whole repo's convention. | +| Collector daemon | `lousho observe` CLI + `src/observe/` module, plain `node:http` | No new framework for a loopback API; the CLI already owns daemon style commands. | +| Primary DB | SQLite via better-sqlite3, rollup tables for Insights | Already the SDK store for sessions; transactional writes plus hand rolled rollups cover local scale analytics. | +| Ingest API | `POST /api/v1/ingest`, batched envelope, loopback only, no auth | Langfuse shaped batching (basis: Langfuse ingestion API); loopback binding matches the node server rule, refuse public bind without a token. | +| Agent wiring | `LOUSHO_OBSERVE` env (`=1` local default or a URL) plus `createAgent({ observability })` | Env first means existing bots emit with zero code changes; the config object stays the explicit override. | +| Dashboard | Panels inside `apps/agent-forge` (React, Vite) | One Studio to build, run, and inspect agents; the app shell and data model already exist. | +| Trace schema | Observations first, denormalized rows, `gen_ai.*` attribute names | The Langfuse v4 pattern (one wide table, trace attrs flattened per row) fits SQLite cleanly; semconv names keep OTLP export a rename away. | +| Sandboxing | Driver interface `sandbox(driver)`; v1 ships `isolated-vm` | A driver seam keeps in process speed now and Docker or E2B sidecars open later (basis: TanStack AI isolate drivers). | +| Hosted path | Same ingest API on a remote endpoint | No SDK change needed to point agents at a hosted collector later. | + +## Cross child contract + +Every child spec builds on these shared shapes. They are the contract between the emitter, the daemon, and Studio. + +**Entities** (single SQLite file, default `~/.lousho/observe.db`): + +| Entity | Rows | Key fields | +|---|---|---| +| `agents` | one per reporting agent | `agent_id` (pk), `project`, `name`, `version`, `first_seen`, `last_heartbeat`, `status` | +| `sessions` | conversation threads | `session_id` (pk), `agent_id` (fk), `started_at`, `ended_at`, `status` | +| `observations` | the wide table | `obs_id` (pk), `trace_id`, `session_id`, `agent_id`, `parent_obs_id`, `kind` (`span`, `generation`, `tool`, `event`), `name`, `start`, `end`, `status`, `input`, `output`, `model`, `usage_*`, `cost_*`, `attrs` (JSON, `gen_ai.*` names) | +| `scores` | eval results | `score_id` (pk), `trace_id`, `obs_id` nullable, `name`, `value` numeric, `source` (`eval`, `judge`, `human`), `comment` | +| `prompts` | registry entries | `prompt_id` (pk), `name`, `version`, `labels`, `text`, `variables`, `created_at`, `hash` | +| `prompt_uses` | run linkage | `obs_id` (fk), `prompt_id` (fk), `version` | +| `hub_entries` | shared memory | `entry_id` (pk), `namespace`, `agent_id` (writer), `key`, `value`, `scope` (`private`, `fleet`), `created_at`, `ttl` | +| `rollups` | Insights aggregates | `bucket_start`, `granularity`, `agent_id`, `metric`, `value` | +| `tokens` | daemon auth keys | `token_id` (pk), `project`, `key_hash`, `created_at`, `revoked_at` | + +Traces are virtual: a trace is the set of `observations` sharing one `trace_id`, with the root span carrying trace level fields flattened onto every row (the Langfuse v4 move (basis: Langfuse v4 observations first model)). + +**Field and id provenance** (decided, not builder invention): + +- `obs_id` is the existing `TraceLine.id` from `src/traces` (never a fresh UUID per flush), so resent batches upsert by key. `updated_at` gives last writer wins on re-emitted rows. +- `session_id` is the agent session's resumable id; reporters outside sessions mint `sha256(agent_id + cwd)` persisted under `.lousho/` so a restarted process rejoins its session stream. +- `agent_id` is a reporter generated UUID persisted at `~/.lousho/identity/@`. +- `usage_*`/`cost_*` carry `StepUsage` from `src/models/usage.ts` verbatim (`usage_input/output/cached/reasoning_tokens`, `cost_usd`, `cost_partial`, `usage_estimated`); the daemon stores them, it never recomputes; the `cost_partial` and `estimated` flags must survive to Studio or cost numbers lie. +- One event per **finished** observation carrying `start` + `end`; in-flight spans are reporter local state (matches `TraceLine` semantics). +- `rollups` is a disposable derivation, not a second write path: recompute bucket rows with `INSERT … SELECT … GROUP BY` inside each ingest transaction (idempotent on resends), plus `lousho observe rebuild-rollups` for repair. +- `observations` is source of truth; WAL mode plus `busy_timeout=5000` at open, single writer daemon. + +**Threat model**: any process on loopback is trusted. `hub` `private` scope is organizational (keeps agents' views clean), not a security boundary. Public binding still requires a token per the node server rule. + +**Ingest API** (the only endpoint that matters; query endpoints are child scope): + +```http +POST /api/v1/ingest +{ "batch": [ + { "type": "agent.heartbeat", "data": { "agent_id": "...", "project": "...", ... } }, + { "type": "agent.deregister", "data": { "agent_id": "..." } }, + { "type": "observation", "data": { "obs_id": "...", "trace_id": "...", ... } }, + { "type": "prompt_use", "data": { "obs_id": "...", "prompt_id": "...", "version": 3 } }, + { "type": "score", "data": { "score_id": "...", "trace_id": "...", ... } } +] } +``` + +Batched, async, idempotent by row primary key (a resent batch upserts). The daemon acks fast and writes off the request path. Payload limits: `input`/`output` strings truncate at about 64KB with a `truncated` flag set; a batch over about 5MB gets `413` and counts into the reporter's `dropped` metric. + +**Read path** (decided, not deferred): Studio is a browser app, it cannot open `~/.lousho/observe.db`. The daemon owns the only database handle and serves every Studio read over `GET /api/v1/traces`, `/traces/:id`, `/insights`, `/agents`, `/scores`, `/prompts`, `/hub`. A "open the db file directly" path is rejected: it breaks the hosted upgrade and the single writer rule. + +**Env contract**: + +- `LOUSHO_OBSERVE`: `1` (or `true`) emits to `http://127.0.0.1:7777`; a URL emits there. Unset emits nothing. +- `LOUSHO_OBSERVE_PROJECT`: project label on every row (default: package name). +- `LOUSHO_OBSERVE_TOKEN`: bearer token, required only when the daemon is configured to demand one. +- `LOUSHO_OBSERVE_PORT` (or `lousho observe --port`): daemon port, default 7777. On `EADDRINUSE` the daemon probes `GET /health` for the `lousho-observe` service marker and attaches or fails with an actionable message. + +**Daemon commands**: `lousho observe start|status|stop`, `lousho observe token create --project X` (writes a hashed key into `tokens`; ingest and read endpoints check it once the daemon runs non loopback), `lousho observe rebuild-rollups`. + +## Consequences + +**Positive**: +- One pane for the whole fleet: Monitoring actually shows every bot on the machine. +- LangSmith shape without hosting: the ingest API is the product seam, so the hosted upgrade is an endpoint swap, not a rewrite. +- Cheap adoption: env var on, everything emits; no per agent code edits. + +**Negative / tradeoffs**: +- A new process to run: the fleet pane needs the daemon up; agents degrade to silent no emit when it is down (by design, but a support question). +- Hand rolled rollups: Insights aggregates live in SQL we own, no OLAP engine to lean on if query needs grow. +- `isolated-vm` is a native dependency: install friction for sandbox users only, kept behind the driver seam so core users never pay it. +- `apps/agent-forge` grows a second job: observability panels beside the builder canvas means the app's data layer must handle a remote data source, not only project files. + +**Neutral**: +- New env vars (`LOUSHO_OBSERVE*`) join the config surface. +- Existing `src/traces` and OTel hooks stay as the low level event source; the reporter consumes them rather than replacing them. + +## Follow-up + +- [ ] Each child subsystem needs its own `/architect` FEATURE pass with real acceptance criteria before `/develop` builds it; this spec fixed the platform, not the per surface contract. +- [ ] `docs/specs/` is not in the docs site `PAGES` map; decide whether spec files belong under docs at all or move to a non published path. +- [ ] DuckDB stays a named upgrade path for Insights if SQLite rollups ever hurt; not in v1. +- [ ] Docker or E2B sandbox driver follows the isolate driver once the seam exists. +- [ ] Decide prompt registry credentials: prompt fetch from the daemon needs no auth on loopback, the hosted path needs keys. + +## Rationale + +Reasoning, options, and the landscape evidence: see [rationale.md](rationale.md). diff --git a/docs/specs/0001-observability-platform/rationale.md b/docs/specs/0001-observability-platform/rationale.md new file mode 100644 index 00000000..633238a2 --- /dev/null +++ b/docs/specs/0001-observability-platform/rationale.md @@ -0,0 +1,99 @@ +# Rationale: observability platform + +## Context + +> ⚠️ Premise note: the request listed ten subsystems (Engine, Studio, Build, Tracing, Insights, Monitoring, Evaluators, Prompts, Context Hub, Sandboxing), which is a product, not one decision. Five of the ten already exist in the SDK (the runtime, Agent Forge Studio, `lousho build`, `src/traces`, `src/evals`). This spec fixes the platform foundation the genuinely new pieces share; each subsystem still needs its own feature spec before build. + +The SDK runs agents inside the user's own Node process. Once agents ship, there is no place to see what they did: which tools ran, what a run cost, where it failed, whether behavior drifted between versions. LangSmith answers this for LangChain by being a hosted service every agent reports to; the SDK's equivalent must work without anyone running infrastructure, because most users will not. + +The engineer runs a fleet of bots on one machine (a companion, a security agent, a business agent, a trading agent). Their real need, a single pane over all of them, already forces a collector that outlives any one project directory. That makes the honest architecture a small local service, not a per project log file. + +## Options considered + +### Option 1: hosted service first + +A new `apps/observatory` server: agents POST runs over API keys, Postgres storage, a web dashboard. The true LangSmith shape. + +**Pros**: +- Works across machines and teams from day one. +- No questions about where data lives; one service owns it. + +**Cons**: +- Ships auth, tenancy, and ops on day one for a feature most users want locally. +- A second product to maintain before the first one has proven its data model. +- Every SDK user needs an account or self hosting just to see a run. + +### Option 2: hybrid platform, local first (chosen) + +An open ingest API plus a local implementation now: `lousho observe` daemon, SQLite, Studio panels. A hosted collector implements the same API later. + +**Pros**: +- Fleet view today (one daemon per machine), hosted view later by repointing `LOUSHO_OBSERVE`. +- The data model and ingest contract get proven locally before they harden into a service API. +- Zero infrastructure for SDK users; matches how sessions already store locally. + +**Cons**: +- The local daemon is one more process; monitoring dies quietly when it is not running. +- The API contract must be designed for the hosted future now (batching, idempotency, project scoping) or the upgrade path rots. + +### Option 3: OTel native, no own surface + +Emit OpenTelemetry only; point users at Jaeger or Grafana. + +**Pros**: +- Nothing new to build or operate on our side. + +**Cons**: +- Generic spans, not agent aware: no sessions, evals, prompt versions, or fleet registry. That is tracing tooling, not a LangSmith clone. +- OTel GenAI semantic conventions are still "Development" status; depending on them alone bakes in churn. + +### Option 4: per project store only + +Each project writes `.lousho/traces.db`, viewed only inside that project's Studio. + +**Pros**: +- No daemon, no network, simplest possible. + +**Cons**: +- Permanently rules out a fleet pane and observing deployed agents; Monitoring as scoped cannot exist. + +## Rationale + +Local scale is small (tens of agents, millions of rows at most), so the storage and analytics problems Langfuse solved with ClickHouse simply do not exist here (basis: landscape check). better-sqlite3 is already the session store dependency, so SQLite with rollup tables is the boring correct pick; DuckDB Neo is now a real embedded OLAP option but a ~30MB binary on a 2.0.0 release is not a v1 dependency for this size. + +The schema decision borrows directly from Langfuse's own migration path: they ended at an observations first denormalized model because trace level queries dominate everything (basis: Langfuse v4 observations first model). One wide table with flattened trace attributes is also exactly what SQLite is good at, and `gen_ai.*` attribute names keep a future OTLP exporter cheap even though the convention is still moving. + +For sandboxing, the durable decision is the driver interface, not the isolate engine. `isolated-vm` works today but is maintenance mode and a native module; the agents world has already converged on pluggable sandbox drivers (isolate for speed, containers or E2B for real isolation). Shipping the seam plus the fast driver mirrors that consensus (basis: TanStack AI isolate driver pattern). + +The loopback only, no auth default follows the repo's own node server rule: refuse to bind publicly without a token rather than protect inside. A token option exists for the hosted path without changing the local default. + +## References + +**Project sources** (verifiable, in this repo): +- `apps/agent-forge`: the existing Studio app the dashboard panels extend. +- `src/traces`, `src/execution`: the run and step event source the reporter consumes. +- `src/evals`: `defineEval`, `llmJudge`, cassettes already produce eval outputs that become score rows. +- `FileSessionStore` and the node server auth rule (`src/server`): the SQLite precedent and the loopback binding convention this spec copies. +- `LOUSHO_API_TOKEN` / `LOUSHO_ALLOW_PUBLIC` pattern: the existing env contract style `LOUSHO_OBSERVE*` follows. + +**Practices & standards**: +- Langfuse data model and ingestion API shape (observations first, batched ingest). +- OpenTelemetry GenAI semantic conventions for attribute names (`gen_ai.*`), used as names only while the convention stays in Development status. +- TanStack AI isolate driver pattern for the sandbox seam. + +**Links** (web verified during the landscape check): +- Langfuse architecture: https://langfuse.com/handbook/product-engineering/architecture +- Langfuse observability data model: https://langfuse.com/docs/observability/data-model +- Langfuse v3 to v4 upgrade (observations first move): https://langfuse.com/self-hosting/upgrade/upgrade-v3-to-v4 +- ClickHouse at agent scale (Langfuse engineering): https://langfuse.com/resources/engineering/clickhouse-at-agent-scale +- isolated-vm repository (status, compat table): https://github.com/laverdet/isolated-vm +- TanStack AI code mode isolates docs: https://tanstack.com/ai (code mode isolates) +- DuckDB Node Neo client: https://github.com/duckdb/duckdb-node-neo and https://duckdb.org/2024/12/18/duckdb-node-neo-client +- OTel GenAI semantic conventions repo: https://github.com/open-telemetry/semantic-conventions-genai + +## Landscape evidence (what the check actually found) + +- DuckDB Neo (`@duckdb/node-api`) is the official, TypeScript native client with prebuilt binaries and ~1.18M weekly downloads; the v2.0.0 line just shipped with the maintainer warning that bugs are likely on the major bump. Pick worthy later, not now. +- `isolated-vm` v7.0.1 tracks current Node majors but is explicitly maintenance mode and carries known isolation limits. `node:vm` is documented as not a security boundary. Agents and eval frameworks mostly use container or microVM sidecars for real untrusted code. +- Langfuse runs Postgres plus ClickHouse plus a worker queue at SaaS scale; their v4 flattened traces into virtual root spans over one wide observations table. That denormalized shape, not their infra, is what transfers to a local store. +- OTel GenAI semconv moved to its own repo and remains Development status; Arize OpenInference is the alternative convention. Adopt names, not the pipeline. diff --git a/llms-full.txt b/llms-full.txt index 289f1fe3..ef003ebd 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4913,6 +4913,8 @@ lousho mcp [--http --port N --host H] lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping] lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--config vitest.config.ts] [--timeout ] lousho traces [traceId|prefix] [--dir D] [--limit N] [--json] [--content] +lousho observe [--port N] [--db path] [--host H] [--require-token] [--project X] [--webhook URL] +lousho prompt |list|get > [--url U] [--port N] [--token T] [--json] [--name N] lousho --version ``` @@ -5143,6 +5145,35 @@ Starts one local server for the Agent Forge API and UI. Agents and run state are stored under `.lousho/` in the current directory. See [Agent Forge](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/agent-forge.md) for the walkthrough. +## `lousho observe` + +```bash +npx lousho observe start # collector on http://127.0.0.1:7777 +npx lousho observe status # health + fleet counts +npx lousho observe stop # pidfile + SIGTERM +npx lousho observe rebuild-rollups # recompute insight aggregates +npx lousho observe token create --project myapp # bearer key for non-loopback auth +``` + +Runs the local collector every agent reports to when `LOUSHO_OBSERVE` is +set: traces, fleet monitoring, eval scores, the prompt registry and the +context hub in one SQLite file (default `~/.lousho/observe.db`). The Studio +Observe tabs read it. `start` takes `--port` (or `LOUSHO_OBSERVE_PORT`), +`--db`, `--host`, `--require-token` and a repeatable `--webhook` for alert +hooks. See [The observe daemon](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md#the-observe-daemon). + +## `lousho prompt` + +```bash +npx lousho prompt push prompt.md --name mira # publishes a new version +npx lousho prompt list +npx lousho prompt get mira@latest +``` + +Publishes and reads prompt versions in the observe daemon's registry. Agents +resolve them with `createAgent({ prompt: 'mira@latest' })` or +`LOUSHO_PROMPT`. See [The observe daemon](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md#the-observe-daemon). + # Cloudflare Workers Source: docs/cloudflare-workers.md @@ -8368,7 +8399,7 @@ Find a code by area: | [Schedules](#schedules) | [`LOUSHO_SCHEDULE_INVALID`](#lousho_schedule_invalid), [`LOUSHO_SCHEDULE_RUN_INCOMPLETE`](#lousho_schedule_run_incomplete) | Defining or loading a schedule, or a scheduled turn that did not finish. | | [Channels](#channels) | [`LOUSHO_CHANNEL_INVALID`](#lousho_channel_invalid), [`LOUSHO_MEMORY_INVALID`](#lousho_memory_invalid) | Defining a channel or a memory slot. | | [Registry](#registry) | [`LOUSHO_REGISTRY_UNREACHABLE`](#lousho_registry_unreachable), [`LOUSHO_REGISTRY_ITEM_NOT_FOUND`](#lousho_registry_item_not_found), [`LOUSHO_REGISTRY_INVALID`](#lousho_registry_invalid), [`LOUSHO_REGISTRY_UNSAFE_PATH`](#lousho_registry_unsafe_path), [`LOUSHO_REGISTRY_FILE_EXISTS`](#lousho_registry_file_exists), [`LOUSHO_REGISTRY_MANIFEST_MISMATCH`](#lousho_registry_manifest_mismatch) | `lousho add` fetching or copying from a registry. | -| [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied) | Asking a sandbox for something it cannot do on the current platform, or to write outside its root. | +| [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied), [`LOUSHO_SANDBOX_FAILED`](#lousho_sandbox_failed) | Asking a sandbox for something it cannot do on the current platform, writing outside its root, or a code-sandbox boundary (timeout / OOM / driver) failing a run. | | [Agent directories, skills and flows](#agent-directories-skills-and-flows) | [`LOUSHO_AGENT_DIR_INVALID`](#lousho_agent_dir_invalid), [`LOUSHO_SKILL_INVALID`](#lousho_skill_invalid), [`LOUSHO_FLOW_INVALID`](#lousho_flow_invalid), [`LOUSHO_FLOW_TOOL_DENIED`](#lousho_flow_tool_denied), [`LOUSHO_FLOW_CHANGED`](#lousho_flow_changed) | Loading an agent directory, a skill, or a flow definition, and gating a flow's tool calls. | | [Storage, deployment and integrations](#storage-deployment-and-integrations) | [`LOUSHO_STORAGE_FAILED`](#lousho_storage_failed), [`LOUSHO_STORAGE_BUSY`](#lousho_storage_busy), [`LOUSHO_TRIGGER_INVALID`](#lousho_trigger_invalid), [`LOUSHO_CHANNEL_REQUEST_FAILED`](#lousho_channel_request_failed), [`LOUSHO_DEPLOY_FAILED`](#lousho_deploy_failed), [`LOUSHO_SERVER_UNPROTECTED`](#lousho_server_unprotected) | A storage backend, a trigger, a channel request, `lousho build` or the built server's start-up. | | [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. | @@ -9234,6 +9265,23 @@ directory at construction). See [Guardrails](https://github.com/LinuxDevil/agent **Example:** `sandbox.writeFile('/etc/profile.d/x.sh', ...)` with the default root. +### LOUSHO_SANDBOX_FAILED + +**Means:** a `sandbox()` / `defineTool({ sandbox })` run failed at the +sandbox boundary - not in the tool's own code, whose errors cross back +unchanged. The `SandboxError`'s `kind` names which boundary: `'timeout'` +(the run outlasted `timeoutMs`), `'oom'` (the driver killed it for its +memory cap - with the `'isolate'` driver this can also take the host +process, since the isolate is an execution boundary, not a security one) +or `'driver'` (the driver itself failed). The tool span carries the same +kind as `sandbox.error`. + +**Fix:** raise `memoryMb`/`timeoutMs`, or fix the driver; for a missing +`isolated-vm` install the optional peer or pass another `SandboxDriver`. + +**Example:** `sandbox({ driver: 'isolate', timeoutMs: 5_000 })` running code +that does not return in five seconds. + ## Agent directories, skills and flows ### LOUSHO_AGENT_DIR_INVALID @@ -13991,6 +14039,39 @@ and prompt and tool content only if the agent sets `redactContent: false` or `captureContent: true`, so nothing is written to disk unless you ask. The spec file format has no traces setting. +## The observe daemon + +`lousho observe start` runs a local collector on `http://127.0.0.1:7777` +(loopback only, no auth needed). When `LOUSHO_OBSERVE` is set, every agent +built on this SDK reports spans to it automatically — `1` (or `true`) emits +to the default URL, a URL value emits there. `LOUSHO_OBSERVE_PROJECT` labels +the rows; `LOUSHO_OBSERVE_TOKEN` sends a bearer token for daemons bound +beyond loopback (`--require-token` plus `lousho observe token create`). + +One SQLite file (default `~/.lousho/observe.db`, WAL mode) holds the fleet's +observations, sessions, eval scores, prompt versions and shared context. +`npx lousho studio` renders it in the Observe tabs: trace trees with latency +waterfalls, per-agent insights (runs/day, p50/p95 latency, cost, error +rate), a live fleet monitor (heartbeat-derived status — a crashed agent's +open sessions flip to `interrupted`), and eval score history. Alerts on +`down` transitions or error-rate thresholds POST to `--webhook` URLs or an +`alerts.json` next to the db. + +The daemon also serves two stores agents can use at runtime: + +- **Prompt registry.** `createAgent({ prompt: 'name@version' })` or + `LOUSHO_PROMPT` fetches instructions from it at build (`@latest` resolves + on the daemon; an unreachable daemon falls back to `instructions`). Every + run logs the exact version used. `lousho prompt push` publishes a new + version. +- **Context hub.** `hub()` / `hubTools()` let agents publish and query + namespaced key-values scoped `private` (the writing agent only) or + `fleet` (every agent on the daemon). + +Reporting is fire-and-forget: a down daemon drops and counts events, it +never blocks a run. Eval scores flow through the same pipe — +`lousho eval --observe` emits them even when `LOUSHO_OBSERVE` is unset. + ## The built server The server `lousho build --target=node-server` or `--target=docker` generates diff --git a/package.json b/package.json index cae3dcda..8861b7a5 100644 --- a/package.json +++ b/package.json @@ -402,6 +402,7 @@ "@opentelemetry/api": "^1.9.1", "ai": "^4.3.19 || ^6.0.0 || ^7.0.0", "dockerode": "^5.0.1", + "isolated-vm": "^6.2.0 || ^7.0.0", "ollama-ai-provider": "^1.2.0", "ollama-ai-provider-v2": "^2.0.0 || ^3.0.0 || ^4.0.0", "prompts": "^2.4.2", @@ -436,6 +437,9 @@ "dockerode": { "optional": true }, + "isolated-vm": { + "optional": true + }, "ollama-ai-provider": { "optional": true }, diff --git a/scripts/size-budget.json b/scripts/size-budget.json index 4e3f3047..213000e2 100644 --- a/scripts/size-budget.json +++ b/scripts/size-budget.json @@ -1,4 +1,4 @@ { - "bundleBytes": 435883, - "tarballUnpackedBytes": 15881428 + "bundleBytes": 473361, + "tarballUnpackedBytes": 17258051 } diff --git a/scripts/typedoc-baseline.json b/scripts/typedoc-baseline.json index 08ed339d..27714537 100644 --- a/scripts/typedoc-baseline.json +++ b/scripts/typedoc-baseline.json @@ -1,3 +1,3 @@ { - "notDocumented": 1777 + "notDocumented": 1758 } diff --git a/src/cli/eval.test.ts b/src/cli/eval.test.ts index e3c7e600..841ebbd0 100644 --- a/src/cli/eval.test.ts +++ b/src/cli/eval.test.ts @@ -124,7 +124,7 @@ describe('runEval with a fake vitest', () => { const log = vi.fn(); expect(await runEval(['--help'], { log })).toBe(0); const help = String(log.mock.calls[0][0]); - for (const flag of ['--tag', '--junit', '--json', '--strict', '--judge', '--record', '--replay', '--drift', '--drift-usage', '--url', '--token', '--config', '--timeout']) { + for (const flag of ['--tag', '--junit', '--json', '--strict', '--judge', '--record', '--replay', '--drift', '--drift-usage', '--url', '--token', '--observe', '--config', '--timeout']) { expect(help).toMatch(new RegExp(`\\n ${flag}\\b[^\\n]* {2,}\\S`)); } }); @@ -214,6 +214,41 @@ describe('runEval with a fake vitest', () => { }); }); +describe('lousho eval --observe', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('parses --observe as a boolean flag', () => { + expect(parseEvalArgs(['--observe'])).toMatchObject({ observe: true }); + expect(parseEvalArgs([]).observe).toBeUndefined(); + }); + + it('points the worker at the local daemon when LOUSHO_OBSERVE is unset or off', async () => { + vi.stubEnv('LOUSHO_OBSERVE', ''); + let fake = fakeVitest([result({})]); + await runEval(['--observe'], { resolveVitest: () => 'v', spawnVitest: fake.spawnVitest, log: () => {} }); + expect(fake.calls[0].env.LOUSHO_OBSERVE).toBe('1'); + + vi.stubEnv('LOUSHO_OBSERVE', '0'); + fake = fakeVitest([result({})]); + await runEval(['--observe'], { resolveVitest: () => 'v', spawnVitest: fake.spawnVitest, log: () => {} }); + expect(fake.calls[0].env.LOUSHO_OBSERVE).toBe('1'); + }); + + it('keeps a configured LOUSHO_OBSERVE URL and does nothing without --observe', async () => { + vi.stubEnv('LOUSHO_OBSERVE', 'http://collector.test:9999'); + let fake = fakeVitest([result({})]); + await runEval(['--observe'], { resolveVitest: () => 'v', spawnVitest: fake.spawnVitest, log: () => {} }); + expect(fake.calls[0].env.LOUSHO_OBSERVE).toBe('http://collector.test:9999'); + + vi.stubEnv('LOUSHO_OBSERVE', ''); + fake = fakeVitest([result({})]); + await runEval([], { resolveVitest: () => 'v', spawnVitest: fake.spawnVitest, log: () => {} }); + expect(fake.calls[0].env.LOUSHO_OBSERVE).toBe(''); + }); +}); + describe('lousho eval --url', () => { it('parses --url and --token', () => { expect(parseEvalArgs(['--url', 'https://a.test', '--token=abc'])).toMatchObject({ url: 'https://a.test', token: 'abc' }); diff --git a/src/cli/eval.ts b/src/cli/eval.ts index 1d4cce27..cc08257c 100644 --- a/src/cli/eval.ts +++ b/src/cli/eval.ts @@ -25,7 +25,7 @@ import { USAGE_EXIT_CODE, parseCommand, stringValue, usageError, type CommandSpe import { failsRun, parseResults, renderDriftTable, renderJson, renderJunit, renderTable, unreportedFailures, vitestError, type DriftRow } from './evalReport'; const USAGE = - 'Usage: lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--config vitest.config.ts] [--timeout ]'; + 'Usage: lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--observe] [--config vitest.config.ts] [--timeout ]'; /** Parsed `lousho eval` arguments. */ export interface EvalCliArgs { @@ -59,6 +59,12 @@ export interface EvalCliArgs { drift?: boolean; /** With `drift`: compare token usage too. */ driftUsage?: boolean; + /** + * Ship eval scores to the observe daemon even when `LOUSHO_OBSERVE` is + * unset (emits to its URL when set, else the local default), so CI runs + * appear in Studio's score history. + */ + observe?: boolean; /** `-h` / `--help` was given: print the usage, run nothing. */ help?: boolean; } @@ -82,6 +88,8 @@ Options: --drift-usage with --drift: compare token usage too --url run the cases against the agent deployed at instead of in-process --token with --url: the bearer token (default LOUSHO_EVAL_TOKEN) + --observe emit eval scores to the observe daemon (LOUSHO_OBSERVE URL when set, + else the local default); on automatically when LOUSHO_OBSERVE is set --config use your own vitest config instead of the generated one --timeout time limit per case (0 = none; live runs default to 10 minutes) -h, --help show this help @@ -106,6 +114,7 @@ const SPEC: CommandSpec = { replay: { type: 'boolean' }, drift: { type: 'boolean' }, 'drift-usage': { type: 'boolean' }, + observe: { type: 'boolean' }, }, }; @@ -155,6 +164,7 @@ export function parseEvalArgs(rest: string[]): EvalCliArgs { replay: v.replay === true || undefined, drift: v.drift === true || undefined, driftUsage: v['drift-usage'] === true || undefined, + observe: v.observe === true || undefined, }; assertOneCassetteMode(args); return args; @@ -247,6 +257,12 @@ export function toRootRelativeGlob(glob: string, cwd: string): string { return relative.split(path.sep).join('/'); } +/** `LOUSHO_OBSERVE` is on unless unset, '0' or 'false' (the reporter's own rule). */ +function observeEnabled(flag: string | undefined): boolean { + const value = flag?.trim(); + return value !== undefined && value !== '' && value !== '0' && value.toLowerCase() !== 'false'; +} + function prepareInvocation(args: EvalCliArgs, cwd: string, workDir: string): VitestInvocation { const resultsFile = path.join(workDir, 'results.jsonl'); fs.writeFileSync(resultsFile, ''); @@ -272,6 +288,9 @@ function prepareInvocation(args: EvalCliArgs, cwd: string, workDir: string): Vit ...(args.token ? { [REMOTE_TOKEN_ENV]: args.token } : {}), [CASSETTES_ENV]: cassetteMode(args), [REMOTE_URL_ENV]: args.url ?? '', + // --observe forces score emission in the worker: a set LOUSHO_OBSERVE (a URL) + // passes through untouched; unset/'0'/'false' gets pointed at the local default. + ...(args.observe && !observeEnabled(process.env.LOUSHO_OBSERVE) ? { LOUSHO_OBSERVE: '1' } : {}), [DRIFT_DIR_ENV]: args.drift ? path.join(workDir, 'drift') : '', [CONFIG_ENV]: args.config ? toRootRelativeGlob(args.config, cwd) : '', }, diff --git a/src/cli/observe.test.ts b/src/cli/observe.test.ts new file mode 100644 index 00000000..911b137a --- /dev/null +++ b/src/cli/observe.test.ts @@ -0,0 +1,108 @@ +/** + * `lousho observe` against a real daemon on an ephemeral port and temp dbs: + * status reports a running collector and misses a dead one, start answers + * "already running" on the daemon's own port and 1 on a real collision, and + * stop/rebuild/token work the local db. The listen-happy-path of `start` + * stays in daemon.test.ts (a started daemon has no reachable handle here). + */ +import * as fs from 'node:fs'; +import * as http from 'node:http'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { defaultPidPath, startObserveDaemon, type ObserveDaemon } from '../observe/daemon'; +import { runObserve } from './observe'; + +const daemons: ObserveDaemon[] = []; +const servers: http.Server[] = []; +let cwd: string; +let logs: string[]; +let errors: string[]; + +beforeEach(() => { + cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-observe-cli-')); + logs = []; + errors = []; + vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => void logs.push(args.map(String).join(' '))); + vi.spyOn(console, 'error').mockImplementation((...args: unknown[]) => void errors.push(args.map(String).join(' '))); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + while (daemons.length > 0) await daemons.pop()?.close(); + for (const server of servers.splice(0)) await new Promise((resolve) => server.close(() => resolve())); + fs.rmSync(cwd, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +async function start(): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0 }); + daemons.push(daemon); + return daemon; +} + +const dbPath = (): string => path.join(cwd, 'observe.db'); + +describe('lousho observe', () => { + it('prints help (0 with --help, 2 with none) and exits 2 on an unknown command', async () => { + expect(await runObserve([])).toBe(2); + expect(logs.at(-1)).toContain('lousho observe'); + expect(await runObserve(['--help'])).toBe(0); + expect(await runObserve(['bogus'])).toBe(2); + expect(errors.at(-1)).toContain('unknown command'); + }); + + it('status reports a running collector and its counts', async () => { + const daemon = await start(); + expect(await runObserve(['status', '--url', `http://127.0.0.1:${daemon.port}`])).toBe(0); + expect(logs.join('\n')).toContain('running at'); + expect(logs.join('\n')).toContain('0 agents'); + }); + + it('status exits 1 when nothing answers', async () => { + expect(await runObserve(['status', '--url', 'http://127.0.0.1:1'])).toBe(1); + expect(errors.at(-1)).toContain('no collector'); + }); + + it('start on the daemon\'s own port reports "already running" and exits 0', async () => { + const daemon = await start(); + expect(await runObserve(['start', '--port', String(daemon.port), '--db', dbPath()])).toBe(0); + expect(logs.at(-1)).toContain('already running'); + }); + + it('start on a port a different service owns exits 1 with an actionable error', async () => { + const alien = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end('{"ok":true}'); + }); + await new Promise((resolve) => alien.listen(0, '127.0.0.1', resolve)); + servers.push(alien); + const port = (alien.address() as { port: number }).port; + expect(await runObserve(['start', '--port', String(port), '--db', dbPath()])).toBe(1); + expect(errors.at(-1)).toContain('taken by another service'); + }); + + it('stop exits 1 without a pidfile and clears a stale one', async () => { + const db = dbPath(); + expect(await runObserve(['stop', '--db', db])).toBe(1); + expect(errors.at(-1)).toContain('no pidfile'); + const pidPath = defaultPidPath(db); + fs.mkdirSync(path.dirname(pidPath), { recursive: true }); + fs.writeFileSync(pidPath, '99999999'); + expect(await runObserve(['stop', '--db', db])).toBe(0); + expect(logs.at(-1)).toContain('already gone'); + expect(fs.existsSync(pidPath)).toBe(false); + }); + + it('rebuild-rollups rebuilds the buckets of a local db', async () => { + expect(await runObserve(['rebuild-rollups', '--db', dbPath()])).toBe(0); + expect(logs.at(-1)).toContain('rebuilt'); + }); + + it('token create prints the key once; a missing --project or verb is a usage error', async () => { + const db = dbPath(); + expect(await runObserve(['token', 'create', '--project', 'smoke', '--db', db])).toBe(0); + expect(logs.some((line) => line.includes('token created'))).toBe(true); + expect(await runObserve(['token', 'create', '--db', db])).toBe(2); + expect(await runObserve(['token', 'bogus', '--project', 'x', '--db', db])).toBe(2); + }); +}); diff --git a/src/cli/observe.ts b/src/cli/observe.ts new file mode 100644 index 00000000..78b3e0b5 --- /dev/null +++ b/src/cli/observe.ts @@ -0,0 +1,329 @@ +/** + * `lousho observe` - the collector daemon command. It owns the single writer + * to `~/.lousho/observe.db` and serves the ingest + Studio read API. + * + * lousho observe start [--port N] [--db path] [--host H] [--require-token] + * lousho observe status [--port N | --url U] + * lousho observe stop [--db path] + * lousho observe rebuild-rollups [--db path] + * lousho observe token create --project X [--db path] + * + * `start` runs in the foreground (like `lousho dev`). On EADDRINUSE it probes + * `GET /health`: a `lousho-observe` answer means the daemon is already + * running (exit 0); anything else is a real port collision. `stop` uses the + * pidfile `start` wrote beside the db. `token create` is a local db op - it + * needs no running daemon - and prints the key once (only its hash is kept). + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { + defaultDbPath, + defaultPidPath, + isObserveDaemon, + probeHealth, + startObserveDaemon, + type HealthProbe, +} from '../observe/daemon'; +import { createToken, hasTokens, openDb, rebuildRollups } from '../observe/store'; +import { OBSERVE_DEFAULT_PORT, OBSERVE_SERVICE } from '../observe/envelope'; +import { VERSION } from '../version'; +import { exitCodeFor, parseCommand, portValue, stringValue, usageError, type CommandSpec } from './args'; + +const USAGE = `Usage: lousho observe [options] + observe start [--port N] [--db path] [--host H] [--require-token] [--webhook URL] + observe status [--port N] [--url U] + observe stop [--db path] + observe rebuild-rollups [--db path] + observe token create --project X [--db path]`; + +const HELP = `${USAGE} + +Runs the observability collector: one SQLite store (default ~/.lousho/observe.db) that every +agent on this machine reports to when LOUSHO_OBSERVE=1, plus the read API Studio queries. + +Options: + --port N listen port (default ${OBSERVE_DEFAULT_PORT}, or LOUSHO_OBSERVE_PORT) + --db path the SQLite file (default ~/.lousho/observe.db) + --host H bind address (default 127.0.0.1; a non-loopback bind requires a token and turns auth on) + --require-token require 'Authorization: Bearer ' on /api/v1/* even on loopback + --webhook URL 'start': POST alert JSON to this URL when an agent goes down or its error + rate spikes (repeatable). Also reads 'alerts.json' beside the db: + {"webhooks": [...], "errorRateThreshold": 0.5} + --url U 'status': the daemon's base URL (default http://127.0.0.1:) + --project X 'token create': the project the token is labeled for + -h, --help show this help`; + +type ObserveSubcommand = 'start' | 'status' | 'stop' | 'rebuild-rollups' | 'token'; + +const SUB_SPECS: Record = { + start: { + command: 'observe start', + usage: 'Usage: lousho observe start [--port N] [--db path] [--host H] [--require-token] [--webhook URL]', + options: { + port: { type: 'string' }, + db: { type: 'string' }, + host: { type: 'string' }, + 'require-token': { type: 'boolean' }, + webhook: { type: 'string', multiple: true }, + }, + }, + status: { + command: 'observe status', + usage: 'Usage: lousho observe status [--port N] [--url U]', + options: { port: { type: 'string' }, url: { type: 'string' } }, + }, + stop: { + command: 'observe stop', + usage: 'Usage: lousho observe stop [--db path]', + options: { db: { type: 'string' } }, + }, + 'rebuild-rollups': { + command: 'observe rebuild-rollups', + usage: 'Usage: lousho observe rebuild-rollups [--db path]', + options: { db: { type: 'string' } }, + }, + token: { + command: 'observe token', + usage: 'Usage: lousho observe token create --project X [--db path]', + positionals: 1, + options: { db: { type: 'string' }, project: { type: 'string' } }, + }, +}; + +const SUBCOMMANDS = new Set(Object.keys(SUB_SPECS)); + +function dbPathOf(value: string | boolean | (string | boolean)[] | undefined): string { + const given = stringValue(value); + return given === undefined ? defaultDbPath() : path.resolve(given); +} + +/** A `multiple: true` string flag's values (a single use still lands as one element). */ +function stringListValue(value: string | boolean | (string | boolean)[] | undefined): string[] { + if (Array.isArray(value)) return value.filter((v): v is string => typeof v === 'string'); + const single = stringValue(value); + return single === undefined ? [] : [single]; +} + +function envPort(): number { + const raw = process.env.LOUSHO_OBSERVE_PORT; + if (raw === undefined) return OBSERVE_DEFAULT_PORT; + const port = Number(raw); + return Number.isInteger(port) && port >= 0 && port <= 65535 ? port : OBSERVE_DEFAULT_PORT; +} + +/* ------------------------------------------------------------------- start */ + +async function runStart(args: string[]): Promise { + const spec = SUB_SPECS.start; + const { values, help } = parseCommand(spec, args); + if (help) { + console.log(HELP); + return 0; + } + const port = portValue(spec, values.port, envPort()); + const dbPath = dbPathOf(values.db); + const host = stringValue(values.host) ?? '127.0.0.1'; + const requireToken = values['require-token'] === true; + const webhooks = stringListValue(values.webhook); + const pidPath = defaultPidPath(dbPath); + + let daemon; + try { + daemon = await startObserveDaemon({ dbPath, port, host, requireToken, pidPath, webhooks }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EADDRINUSE') { + const health = await probeHealth(port, host); + if (isObserveDaemon(health)) { + console.log(`lousho observe: already running on http://${host}:${port} (service ${OBSERVE_SERVICE}, version ${health.version ?? 'unknown'}).`); + return 0; + } + console.error( + `lousho observe: port ${port} is taken by another service (its /health is not ${OBSERVE_SERVICE}). Pass --port , or stop that service.` + ); + return 1; + } + throw error; + } + + console.log(`lousho observe: ${OBSERVE_SERVICE} v${VERSION} listening on http://${host}:${daemon.port}`); + console.log(`lousho observe: db ${dbPath}`); + if (daemon.requireToken) { + console.log( + hasTokens(daemon.db) + ? 'lousho observe: bearer auth is ON; reporters need LOUSHO_OBSERVE_TOKEN=.' + : `lousho observe: bearer auth is ON but no tokens exist yet - create one: lousho observe token create --project ${dbPath === defaultDbPath() ? '' : ` --db ${dbPath}`}` + ); + } else { + console.log('lousho observe: agents emit with LOUSHO_OBSERVE=1 (no auth on loopback).'); + } + if (daemon.alerts.webhooks.length > 0 || daemon.alerts.errorRateThreshold !== undefined) { + const parts = [ + daemon.alerts.webhooks.length > 0 + ? `${daemon.alerts.webhooks.length} alert webhook${daemon.alerts.webhooks.length === 1 ? '' : 's'}` + : 'no alert webhooks', + daemon.alerts.errorRateThreshold !== undefined + ? `error-rate alert at ${Math.round(daemon.alerts.errorRateThreshold * 100)}%` + : undefined, + ].filter(Boolean); + console.log(`lousho observe: alerts on - ${parts.join(', ')}.`); + } + // The listening server keeps the process alive, like `lousho dev`. + return 0; +} + +/* ------------------------------------------------------------------ status */ + +async function runStatus(args: string[]): Promise { + const spec = SUB_SPECS.status; + const { values, help } = parseCommand(spec, args); + if (help) { + console.log(HELP); + return 0; + } + const port = portValue(spec, values.port, envPort()); + const base = stringValue(values.url) ?? `http://127.0.0.1:${port}`; + let health: HealthProbe | undefined; + try { + const res = await fetch(`${base}/health`, { signal: AbortSignal.timeout(2000) }); + health = res.ok ? ((await res.json()) as HealthProbe) : undefined; + } catch { + health = undefined; + } + if (!isObserveDaemon(health)) { + console.error(`lousho observe: no collector at ${base} (start it with 'lousho observe start').`); + return 1; + } + console.log(`lousho observe: running at ${base} (version ${health.version ?? 'unknown'})`); + try { + const res = await fetch(`${base}/api/v1/status`, { signal: AbortSignal.timeout(2000) }); + if (res.ok) { + const status = (await res.json()) as { counts?: Record }; + const counts = status.counts ?? {}; + console.log( + ` ${counts.agents ?? 0} agents · ${counts.traces ?? 0} traces · ${counts.observations ?? 0} observations · ` + + `${counts.sessions ?? 0} sessions · ${counts.scores ?? 0} scores` + ); + } else { + console.log(' (read endpoints are token-gated; counts not shown)'); + } + } catch { + // status already reported; counts are best effort + } + return 0; +} + +/* -------------------------------------------------------------------- stop */ + +async function runStop(args: string[]): Promise { + const spec = SUB_SPECS.stop; + const { values, help } = parseCommand(spec, args); + if (help) { + console.log(HELP); + return 0; + } + const dbPath = dbPathOf(values.db); + const pidPath = defaultPidPath(dbPath); + let pid: number | undefined; + try { + pid = Number(fs.readFileSync(pidPath, 'utf8').trim()); + } catch { + pid = undefined; + } + if (pid === undefined || !Number.isInteger(pid) || pid <= 0) { + console.error( + `lousho observe: no pidfile at ${pidPath}. If a daemon is running, stop it with Ctrl+C (or 'kill ' / Task Manager).` + ); + return 1; + } + try { + process.kill(pid, 'SIGTERM'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') { + try { + fs.rmSync(pidPath, { force: true }); + } catch { + // stale pidfile already gone + } + console.log(`lousho observe: process ${pid} is already gone; removed the stale pidfile.`); + return 0; + } + throw error; + } + console.log(`lousho observe: sent SIGTERM to pid ${pid}.`); + return 0; +} + +/* ------------------------------------------------------- rebuild / tokens */ + +async function runRebuild(args: string[]): Promise { + const spec = SUB_SPECS['rebuild-rollups']; + const { values, help } = parseCommand(spec, args); + if (help) { + console.log(HELP); + return 0; + } + const db = openDb(dbPathOf(values.db)); + try { + const { buckets } = rebuildRollups(db); + console.log(`lousho observe: rebuilt ${buckets} rollup buckets.`); + return 0; + } finally { + db.close(); + } +} + +async function runToken(args: string[]): Promise { + const spec = SUB_SPECS.token; + const { values, positionals, help } = parseCommand(spec, args); + if (help) { + console.log(HELP); + return 0; + } + if (positionals[0] !== 'create') throw usageError(spec, `expected 'create', got '${positionals[0] ?? ''}'.`); + const project = stringValue(values.project); + if (!project) throw usageError(spec, '--project is required (the label the token is created for).'); + const db = openDb(dbPathOf(values.db)); + try { + const { key } = createToken(db, project); + console.log(`lousho observe: token created for project '${project}'.`); + console.log(key); + console.log('Save it now - only its hash is stored, it cannot be shown again. Reporters send it as LOUSHO_OBSERVE_TOKEN.'); + return 0; + } finally { + db.close(); + } +} + +/** + * CLI entry point. Resolves with the exit code: 0 ok, 1 the subcommand ran and + * failed (not running, real port collision), 2 bad arguments. + */ +export async function runObserve(rest: string[]): Promise { + const [sub, ...subArgs] = rest; + if (sub === undefined || !SUBCOMMANDS.has(sub)) { + if (sub === '--help' || sub === '-h' || sub === undefined) { + console.log(HELP); + return sub === undefined ? 2 : 0; + } + console.error(`lousho observe: unknown command '${sub}'.\n${USAGE}`); + return 2; + } + try { + switch (sub as ObserveSubcommand) { + case 'start': + return await runStart(subArgs); + case 'status': + return await runStatus(subArgs); + case 'stop': + return await runStop(subArgs); + case 'rebuild-rollups': + return await runRebuild(subArgs); + case 'token': + return await runToken(subArgs); + } + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + return exitCodeFor(error); + } +} diff --git a/src/cli/prompt.test.ts b/src/cli/prompt.test.ts new file mode 100644 index 00000000..0fa9f709 --- /dev/null +++ b/src/cli/prompt.test.ts @@ -0,0 +1,115 @@ +/** + * `lousho prompt` against a real daemon on an ephemeral port: push publishes + * a version (from a file, with the basename defaulting the name, or stdin), + * list shows the registry, get resolves name[@version]. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { startObserveDaemon, type ObserveDaemon } from '../observe/daemon'; +import { runPrompt, type PromptDeps } from './prompt'; + +const daemons: ObserveDaemon[] = []; +let cwd: string; +let out: string[]; + +beforeEach(() => { + cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-prompt-cli-')); + out = []; +}); + +afterEach(async () => { + while (daemons.length > 0) await daemons.pop()?.close(); + fs.rmSync(cwd, { recursive: true, force: true }); +}); + +async function start(): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0 }); + daemons.push(daemon); + return daemon; +} + +const base = (daemon: ObserveDaemon): string => `http://127.0.0.1:${daemon.port}`; + +function deps(over: Partial = {}): PromptDeps { + return { cwd, env: {} as NodeJS.ProcessEnv, log: (text) => out.push(text), readStdin: async () => 'from stdin', ...over }; +} + +describe('lousho prompt', () => { + it('push publishes a file as the next version (basename name default), get and list read it back', async () => { + const daemon = await start(); + const file = path.join(cwd, 'mira.txt'); + fs.writeFileSync(file, 'You are Mira.'); + const d = deps(); + + expect(await runPrompt(['push', 'mira.txt', '--url', base(daemon)], d)).toBe(0); + expect(out.at(-1)).toContain('mira@1'); + + expect(await runPrompt(['list', '--url', base(daemon)], d)).toBe(0); + expect(out.at(-1)).toContain('mira'); + expect(out.at(-1)).toContain('@1'); + + expect(await runPrompt(['get', 'mira@1', '--url', base(daemon)], d)).toBe(0); + expect(out.at(-1)).toBe('You are Mira.'); + + // a changed push bumps the version; a bare name resolves latest + fs.writeFileSync(file, 'You are Mira v2.'); + await runPrompt(['push', 'mira.txt', '--url', base(daemon)], d); + expect(await runPrompt(['get', 'mira', '--url', base(daemon)], d)).toBe(0); + expect(out.at(-1)).toBe('You are Mira v2.'); + }); + + it("push '-' reads stdin and requires --name", async () => { + const daemon = await start(); + const d = deps(); + expect(await runPrompt(['push', '-', '--name', 'stdin-prompt', '--url', base(daemon)], d)).toBe(0); + expect(out.at(-1)).toContain('stdin-prompt@1'); + expect(await runPrompt(['get', 'stdin-prompt', '--url', base(daemon)], d)).toBe(0); + expect(out.at(-1)).toBe('from stdin'); + + expect(await runPrompt(['push', '-', '--url', base(daemon)], d)).toBe(2); + }); + + it('push --labels/--variables are stored with the version; get --json shows them', async () => { + const daemon = await start(); + fs.writeFileSync(path.join(cwd, 'p.txt'), 'text'); + const d = deps(); + await runPrompt(['push', 'p.txt', '--name', 'p', '--labels', 'prod,canary', '--variables', 'who', '--url', base(daemon)], d); + expect(await runPrompt(['get', 'p', '--json', '--url', base(daemon)], d)).toBe(0); + const row = JSON.parse(out.at(-1) ?? '{}') as { labels: string[]; variables: string[] }; + expect(row.labels).toEqual(['prod', 'canary']); + expect(row.variables).toEqual(['who']); + }); + + it('exits 1 for a missing prompt and for an unreachable daemon', async () => { + const daemon = await start(); + expect(await runPrompt(['get', 'nope', '--url', base(daemon)], deps())).toBe(1); + expect(await runPrompt(['list', '--url', 'http://127.0.0.1:1'], deps())).toBe(1); + }); + + it('finds the daemon via --port or LOUSHO_OBSERVE_PORT when --url is absent', async () => { + const daemon = await start(); + fs.writeFileSync(path.join(cwd, 'p.txt'), 'text'); + expect(await runPrompt(['push', 'p.txt', '--name', 'p', '--port', String(daemon.port)], deps())).toBe(0); + const d = deps({ env: { LOUSHO_OBSERVE_PORT: String(daemon.port) } as NodeJS.ProcessEnv }); + expect(await runPrompt(['get', 'p'], d)).toBe(0); + // an unparseable env port falls back to the default (7777) rather than crashing + const urls: string[] = []; + const bad = deps({ + env: { LOUSHO_OBSERVE_PORT: 'nope' } as NodeJS.ProcessEnv, + fetchFn: (async (input: RequestInfo | URL) => { + urls.push(String(input)); + throw new Error('fetch failed'); + }) as typeof fetch, + }); + expect(await runPrompt(['list'], bad)).toBe(1); + expect(urls[0]).toContain('127.0.0.1:7777'); + }); + + it('usage errors exit 2; --help prints the help', async () => { + expect(await runPrompt(['bogus'], deps())).toBe(2); + expect(await runPrompt(['--help'], deps())).toBe(0); + expect(out.at(-1)).toContain('lousho prompt'); + }); +}); diff --git a/src/cli/prompt.ts b/src/cli/prompt.ts new file mode 100644 index 00000000..11f2f3a6 --- /dev/null +++ b/src/cli/prompt.ts @@ -0,0 +1,260 @@ +/** + * `lousho prompt` - the prompt-registry command. It talks to a running + * collector daemon (`lousho observe start`), never to the SQLite file. + * + * lousho prompt push --name N [--labels a,b] [--variables x,y] + * lousho prompt list [--json] + * lousho prompt get [--json] + * + * `push` reads the prompt text from a file (`-` = stdin) and publishes a new + * version; an identical republish is deduped daemon side (`created: false`). + * `list` shows one row per name; `get` prints a version's text - `@version` + * is an integer or `latest` (resolved daemon side, like the agents' fetch). + * + * The daemon is found at `--url`, else `LOUSHO_OBSERVE`'s resolution, else + * `http://127.0.0.1:<--port|LOUSHO_OBSERVE_PORT|7777>`; `--token` / + * `LOUSHO_OBSERVE_TOKEN` authenticates when the daemon requires it. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { fetchPrompt, listPrompts, parsePromptRef, publishPrompt, type PromptRef } from '../observe/promptClient'; +import { OBSERVE_DEFAULT_PORT } from '../observe/envelope'; +import { exitCodeFor, parseCommand, portValue, stringValue, usageError, type CommandSpec } from './args'; + +const USAGE = `Usage: lousho prompt [options] + prompt push --name N [--labels a,b] [--variables x,y] [--url U | --port N] [--token T] + prompt list [--url U | --port N] [--token T] [--json] + prompt get [--url U | --port N] [--token T] [--json]`; + +const HELP = `${USAGE} + +Manages the prompt registry a 'lousho observe' collector serves. Agents run with +createAgent({ prompt: 'name@version' }) or LOUSHO_PROMPT=name@version and fetch their +instructions here at build (falling back to embedded instructions when the daemon is down). + +Options: + --name N 'push': the registry name (default: the file's basename, e.g. prompts/mira.txt -> mira; required for '-') + --labels a,b 'push': comma-separated labels stored with the version + --variables x,y 'push': comma-separated variable names stored with the version + --url U the daemon's base URL (default: LOUSHO_OBSERVE's resolution, else http://127.0.0.1:) + --port N the daemon's port when no URL is given (default ${OBSERVE_DEFAULT_PORT}, or LOUSHO_OBSERVE_PORT) + --token T bearer token (default LOUSHO_OBSERVE_TOKEN) for a daemon that requires one + --json 'list' and 'get': print the raw JSON + -h, --help show this help`; + +type PromptSubcommand = 'push' | 'list' | 'get'; + +const DAEMON_OPTIONS = { + url: { type: 'string' }, + port: { type: 'string' }, + token: { type: 'string' }, +} as const; + +const SUB_SPECS: Record = { + push: { + command: 'prompt push', + usage: 'Usage: lousho prompt push --name N [--labels a,b] [--variables x,y] [--url U | --port N] [--token T]', + positionals: 1, + options: { ...DAEMON_OPTIONS, name: { type: 'string' }, labels: { type: 'string' }, variables: { type: 'string' } }, + }, + list: { + command: 'prompt list', + usage: 'Usage: lousho prompt list [--url U | --port N] [--token T] [--json]', + options: { ...DAEMON_OPTIONS, json: { type: 'boolean' } }, + }, + get: { + command: 'prompt get', + usage: 'Usage: lousho prompt get [--url U | --port N] [--token T] [--json]', + positionals: 1, + options: { ...DAEMON_OPTIONS, json: { type: 'boolean' } }, + }, +}; + +const SUBCOMMANDS = new Set(Object.keys(SUB_SPECS)); + +export interface PromptDeps { + /** Where output goes. Default `console.log`. */ + log?: (text: string) => void; + /** Injectable env (tests); default `process.env`. */ + env?: NodeJS.ProcessEnv; + /** Injectable transport (tests); default the global `fetch`. */ + fetchFn?: typeof fetch; + /** `push -`: reads stdin. Default reads the real stdin. */ + readStdin?: () => Promise; + /** Directory relative `push ` paths resolve against (default `process.cwd()`). */ + cwd?: string; +} + +/** The daemon base URL for these commands: --url, else LOUSHO_OBSERVE, else loopback on --port/env/7777. */ +function daemonUrl(values: Record, spec: CommandSpec, env: NodeJS.ProcessEnv): string { + const url = stringValue(values.url); + if (url !== undefined) return url.replace(/\/+$/, ''); + const port = portValue(spec, values.port, envPort(env)); + const flag = env.LOUSHO_OBSERVE?.trim(); + if (flag && flag !== '0' && flag.toLowerCase() !== 'false' && flag !== '1' && flag.toLowerCase() !== 'true') { + return flag.replace(/\/+$/, ''); + } + return `http://127.0.0.1:${port}`; +} + +function envPort(env: NodeJS.ProcessEnv): number { + const raw = env.LOUSHO_OBSERVE_PORT; + if (raw === undefined) return OBSERVE_DEFAULT_PORT; + const port = Number(raw); + return Number.isInteger(port) && port >= 0 && port <= 65535 ? port : OBSERVE_DEFAULT_PORT; +} + +function tokenOf(values: Record, env: NodeJS.ProcessEnv): string | undefined { + return stringValue(values.token) ?? env.LOUSHO_OBSERVE_TOKEN; +} + +function csvOf(value: string | boolean | (string | boolean)[] | undefined): string[] | undefined { + const raw = stringValue(value); + const parts = raw?.split(',').map((part) => part.trim()).filter(Boolean); + return parts !== undefined && parts.length > 0 ? parts : undefined; +} + +/** `get` takes `name[@version]`; a bare name resolves `latest` like an agent's fetch. */ +function refOf(spec: CommandSpec, given: string): PromptRef { + const at = given.indexOf('@'); + if (at === -1) return { name: given, version: 'latest' }; + const parsed = parsePromptRef(given); + if (parsed === undefined) throw usageError(spec, `'${given}' is not a valid name@version (version is an integer or 'latest').`); + return parsed; +} + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString('utf8'); +} + +async function runPush(args: string[], deps: Required): Promise { + const spec = SUB_SPECS.push; + const { values, positionals, help } = parseCommand(spec, args); + if (help) { + deps.log(HELP); + return 0; + } + const file = positionals[0]; + const text = file === '-' ? await deps.readStdin() : fs.readFileSync(path.resolve(deps.cwd, file), 'utf8'); + let name = stringValue(values.name); + if (name === undefined && file !== '-') name = path.basename(file).replace(/\.[^.]+$/, ''); + if (name === undefined) throw usageError(spec, "--name is required when pushing stdin ('-'); for a file it defaults to the basename."); + const base = daemonUrl(values, spec, deps.env); + const { prompt, created } = await publishPrompt( + base, + { name, text, ...(csvOf(values.labels) !== undefined ? { labels: csvOf(values.labels) } : {}), ...(csvOf(values.variables) !== undefined ? { variables: csvOf(values.variables) } : {}) }, + { ...(tokenOf(values, deps.env) !== undefined ? { token: tokenOf(values, deps.env) } : {}), fetchFn: deps.fetchFn } + ); + deps.log(created ? `lousho prompt: published ${prompt.name}@${prompt.version}.` : `lousho prompt: ${prompt.name}@${prompt.version} is unchanged (same text).`); + return 0; +} + +async function runList(args: string[], deps: Required): Promise { + const spec = SUB_SPECS.list; + const { values, help } = parseCommand(spec, args); + if (help) { + deps.log(HELP); + return 0; + } + const base = daemonUrl(values, spec, deps.env); + const prompts = await listPrompts( + base, + {}, + { ...(tokenOf(values, deps.env) !== undefined ? { token: tokenOf(values, deps.env) } : {}), fetchFn: deps.fetchFn } + ); + if (values.json === true) { + deps.log(JSON.stringify(prompts, null, 2)); + return 0; + } + if (prompts.length === 0) { + deps.log('No prompts in the registry. Publish one with `lousho prompt push --name `.'); + return 0; + } + const header = ['NAME', 'LATEST', 'VERSIONS', 'USED BY RUNS', 'UPDATED']; + const rows = prompts.map((prompt) => [ + prompt.name, + `@${prompt.version}`, + String(prompt.versions), + String(prompt.uses), + new Date(prompt.created_at).toISOString(), + ]); + const widths = header.map((_, col) => Math.max(...[header, ...rows].map((row) => row[col].length))); + deps.log([header, ...rows].map((row) => row.map((cell, c) => cell.padEnd(widths[c])).join(' ').trimEnd()).join('\n')); + return 0; +} + +async function runGet(args: string[], deps: Required): Promise { + const spec = SUB_SPECS.get; + const { values, positionals, help } = parseCommand(spec, args); + if (help) { + deps.log(HELP); + return 0; + } + const ref = refOf(spec, positionals[0]); + const base = daemonUrl(values, spec, deps.env); + const prompt = await fetchPrompt( + base, + ref, + { ...(tokenOf(values, deps.env) !== undefined ? { token: tokenOf(values, deps.env) } : {}), fetchFn: deps.fetchFn } + ); + deps.log(values.json === true ? JSON.stringify(prompt, null, 2) : (prompt.text ?? '')); + return 0; +} + +/** Turns a transport failure into the CLI's usual "is the daemon up?" message. */ +function describe(error: unknown, url: string): string { + const message = error instanceof Error ? error.message : String(error); + if (/fetch failed|ECONNREFUSED|ENOTFOUND|timed out/i.test(message)) { + return `lousho prompt: no collector at ${url} (start it with 'lousho observe start', or pass --url).`; + } + return message; +} + +/** The URL the failed subcommand aimed at, best-effort (parse may itself have failed). */ +function urlGuess(sub: PromptSubcommand, args: string[], env: NodeJS.ProcessEnv): string { + try { + const spec = SUB_SPECS[sub]; + return daemonUrl(parseCommand(spec, args).values, spec, env); + } catch { + return `http://127.0.0.1:${envPort(env)}`; + } +} + +/** + * CLI entry point. Resolves with the exit code: 0 ok, 1 the command ran and + * failed (daemon down, prompt missing, bad body), 2 bad arguments. + */ +export async function runPrompt(rest: string[], deps: PromptDeps = {}): Promise { + const resolved: Required = { + log: deps.log ?? ((text) => console.log(text)), + env: deps.env ?? process.env, + fetchFn: deps.fetchFn ?? fetch, + readStdin: deps.readStdin ?? readStdin, + cwd: deps.cwd ?? process.cwd(), + }; + const [sub, ...subArgs] = rest; + if (sub === undefined || !SUBCOMMANDS.has(sub)) { + if (sub === '--help' || sub === '-h' || sub === undefined) { + resolved.log(HELP); + return sub === undefined ? 2 : 0; + } + console.error(`lousho prompt: unknown command '${sub}'.\n${USAGE}`); + return 2; + } + try { + switch (sub as PromptSubcommand) { + case 'push': + return await runPush(subArgs, resolved); + case 'list': + return await runList(subArgs, resolved); + case 'get': + return await runGet(subArgs, resolved); + } + } catch (error) { + console.error(describe(error, urlGuess(sub as PromptSubcommand, subArgs, resolved.env))); + return exitCodeFor(error); + } +} diff --git a/src/createAgent.ts b/src/createAgent.ts index d5295d1b..163f9c5a 100644 --- a/src/createAgent.ts +++ b/src/createAgent.ts @@ -20,6 +20,8 @@ import { AgentExecutor, ExecuteOptions, ExecutionResult } from './execution/Agen import { streamResumed, throwingRun, type AgentRun } from './execution/agentRun'; import type { AgentEvent } from './execution/agentEvents'; import type { TraceExporter } from './execution/tracing'; +import { withObserveExporter, type PromptUseProvider } from './observe/reporter'; +import { parsePromptRef, registryPrompt } from './observe/promptClient'; import { assertModelSettings, mergeModelSettings } from './execution/modelSettings'; import { assertKnownCreateAgentKeys } from './createAgentKeys'; import { DEFAULT_AGENT_NAME } from './execution/genAiSpans'; @@ -607,23 +609,37 @@ export interface CreateAgentBase; - /** @deprecated Use `instructions`. Do not pass both. */ - prompt?: undefined; + /** A `'name@version'` registry ref (`version` an integer or `latest`, resolved daemon side). */ + prompt?: PromptRefString; } | { - /** Preferred name of `prompt`; do not pass both. */ instructions?: undefined; - /** @deprecated Use `instructions`; `prompt` is a legacy alias for it. */ + /** @deprecated Use `instructions`; a non-`'name@version'` `prompt` is a legacy alias for it. */ prompt?: PerRun; }; @@ -1017,7 +1033,11 @@ export function createAgent // Also for resumed runs and when this agent is a sub-agent. ...codeModeOption(config.codeMode), }; - const specs = agentSpecs(config, toolsFor, runOptions); + // The prompt-registry opt-in (config.prompt as 'name@version', or + // LOUSHO_PROMPT) starts its daemon fetch HERE - at agent build, per the + // spec - and the resolved instructions below await it per run. + const resolvedInstructions = agentInstructions(config); + const specs = agentSpecs(config, toolsFor, runOptions, resolvedInstructions); const staticSpec = specs.static; if (staticSpec && config.subagents) assertNoTaskTool(staticSpec.agent, staticSpec.toolRegistry); // Like `task`: a tool named `load_skill` clashes with `skills` here, not at the first send(). @@ -1043,8 +1063,15 @@ export function createAgent ); const hooks = agentHooks(config); // Every run of this agent (send, stream, sessions, resume, approvals) is traced. + // `LOUSHO_OBSERVE` fans the config's exporter out with the collector reporter; + // unset means the env half is a no-op (see src/observe/reporter.ts). + const exporter = withObserveExporter( + config.exporter, + agentName, + resolvedInstructions.promptUse !== undefined ? { promptUse: resolvedInstructions.promptUse } : {} + ); const tracing: Pick = { - ...(config.exporter && { exporter: config.exporter }), + ...(exporter && { exporter }), ...(config.captureContent !== undefined && { captureContent: config.captureContent }), ...(config.redactContent !== undefined && { redactContent: config.redactContent }), }; @@ -1481,8 +1508,13 @@ interface AgentSpecs { resolve: (ctx: RunConfigContext, pinned?: PinnedRunConfig) => Promise; } -function agentSpecs(config: CreateAgentConfig, toolsFor: (tools: AgentToolsOption | undefined) => RunTools, runOptions: RunOptions): AgentSpecs { - const instructions = instructionsOption(config); +function agentSpecs( + config: CreateAgentConfig, + toolsFor: (tools: AgentToolsOption | undefined) => RunTools, + runOptions: RunOptions, + resolved: AgentInstructions +): AgentSpecs { + const { instructions, option } = resolved; const projectBlock = projectInstructionsBlock(config.projectInstructions); const provider = isPerRun(config.model) ? undefined : resolveModelSource(config, config.model); const staticTools = isPerRun(config.tools) ? undefined : toolsFor(config.tools); @@ -1505,7 +1537,7 @@ function agentSpecs(config: CreateAgentConfig, toolsFor: (tools: AgentToolsOptio }; const resolve = async (ctx: RunConfigContext, pinned?: PinnedRunConfig): Promise => { const model = pinned ? pinned.model : await resolveOption('model', config.model, ctx); - const prompt = await resolveOption(config.prompt === undefined ? 'instructions' : 'prompt', instructions, ctx); + const prompt = await resolveOption(option, instructions, ctx); const tools = staticTools ?? toolsFor(await resolveOption('tools', config.tools, ctx)); const spec = specOf(prompt, model, provider ?? resolveModelSource(config, model), tools); spec.agent.metadata = { [RUN_CONFIG_KEY]: { ctx, model } satisfies PinnedRunConfig }; @@ -1705,16 +1737,55 @@ ${found.content}`; /** Used when neither `instructions` nor `prompt` is given. */ const DEFAULT_INSTRUCTIONS = 'You are a helpful assistant.'; -function instructionsOption(config: CreateAgentConfig): PerRun | undefined { - if (config.instructions !== undefined && config.prompt !== undefined) { +/** The instructions the agent resolves per run, and how they got there. */ +interface AgentInstructions { + /** `PerRun` the run resolves: fetched registry text, the embedded value, or a function of the run (undefined = the default). */ + instructions: PerRun | undefined; + /** The option name resolver errors blame ('prompt' only when the legacy alias carries the instructions). */ + option: 'instructions' | 'prompt'; + /** The reporter's `prompt_use` provider when a registry prompt is in play. */ + promptUse?: PromptUseProvider; +} + +/** + * The system prompt the agent runs with: embedded `instructions` (or the + * deprecated `prompt` alias), UNLESS a registry ref is in play - + * `createAgent({ prompt: 'name@version' })`, or `LOUSHO_PROMPT` when + * `prompt` is unset. Then the daemon's text wins and the embedded value is + * the fallback for an unreachable daemon (`LOUSHO_OBSERVE` names it; the + * fetch starts at build and is memoized - see promptClient.registryPrompt). + * + * An explicit `prompt` always beats `LOUSHO_PROMPT`; both `instructions` + * and `prompt` together are an error unless `prompt` is a registry ref + * (then `instructions` is the documented fallback). + */ +function agentInstructions(config: CreateAgentConfig): AgentInstructions { + const promptIsRef = typeof config.prompt === 'string' && parsePromptRef(config.prompt) !== undefined; + if (config.instructions !== undefined && config.prompt !== undefined && !promptIsRef) { throw new ConfigurationError( "createAgent: both 'instructions' and 'prompt' were given. They are the same option - " + - "use 'instructions' (and drop 'prompt', its alias).", + "use 'instructions' (and drop 'prompt', its alias - it only differs from it as a 'name@version' registry reference).", 'prompt', 'LOUSHO_CONFIG_CONFLICTING_OPTIONS' ); } - return config.instructions ?? config.prompt; + const env = typeof process !== 'undefined' ? process.env : undefined; + const ref = promptIsRef ? (config.prompt as string) : config.prompt === undefined ? env?.LOUSHO_PROMPT : undefined; + const fetched = ref !== undefined ? registryPrompt(ref, { env }) : undefined; + if (ref !== undefined && fetched === undefined) { + console.warn( + `[lousho] prompt: '${ref}' needs a collector to resolve - set LOUSHO_OBSERVE=1 (or a daemon URL); the embedded instructions are used instead.` + ); + } + const embedded = promptIsRef ? config.instructions : (config.instructions ?? config.prompt); + if (fetched === undefined) { + return { instructions: embedded, option: config.prompt === undefined ? 'instructions' : 'prompt' }; + } + return { + option: 'instructions', + promptUse: fetched.use, + instructions: (ctx) => fetched.text(() => (embedded === undefined ? Promise.resolve(undefined) : resolveOption('instructions', embedded, ctx))), + }; } /** The default `retry` for models given as strings: as many retries as the `ai` SDK makes on its own. */ diff --git a/src/createAgentPrompt.test.ts b/src/createAgentPrompt.test.ts new file mode 100644 index 00000000..7e8728ea --- /dev/null +++ b/src/createAgentPrompt.test.ts @@ -0,0 +1,128 @@ +/** + * The `prompt` registry opt-in in createAgent(): 'name@version' (or + * LOUSHO_PROMPT) fetches the run's instructions from the observe daemon at + * build, embedded `instructions` are the fallback for an unreachable daemon, + * and a landing fetch links every run to its version through a `prompt_use` + * row (obs_id = the run's root span). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAgent } from './createAgent'; +import { startObserveDaemon, type ObserveDaemon } from './observe/daemon'; +import { publishPrompt } from './observe/promptClient'; +import { observeExporterFromEnv } from './observe/reporter'; +import { mockModel, type MockModel } from './testing'; +import { textOf } from './providers/content'; + +const daemons: ObserveDaemon[] = []; + +beforeEach(() => { + vi.stubEnv('LOUSHO_OBSERVE_PROJECT', 'prompt-test'); +}); + +afterEach(async () => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + while (daemons.length > 0) await daemons.pop()?.close(); +}); + +async function start(): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0 }); + daemons.push(daemon); + return daemon; +} + +const base = (daemon: ObserveDaemon): string => `http://127.0.0.1:${daemon.port}`; + +const systemPrompt = (model: MockModel, call = 0): string => textOf(model.calls[call].messages[0].content); + +/** Flush + release the memoized env reporter for `name` (it is the only path prompt_use events take). */ +async function closeReporter(name: string, url: string): Promise { + const env = { LOUSHO_OBSERVE: url, LOUSHO_OBSERVE_PROJECT: 'prompt-test' } as NodeJS.ProcessEnv; + await observeExporterFromEnv({ agentName: name, env })?.close(); +} + +describe('createAgent({ prompt }) - registry fetch', () => { + it('runs with the fetched registry text instead of the embedded instructions', async () => { + const daemon = await start(); + vi.stubEnv('LOUSHO_OBSERVE', base(daemon)); + await publishPrompt(base(daemon), { name: 'mira', text: 'You are Mira from the registry.' }); + const model = mockModel(['ok']); + + const agent = createAgent({ name: 'prompt-agent-a', provider: model, prompt: 'mira@latest', instructions: 'You are embedded.' }); + await agent.send('hi'); + await closeReporter('prompt-agent-a', base(daemon)); + + expect(systemPrompt(model)).toContain('You are Mira from the registry.'); + expect(systemPrompt(model)).not.toContain('You are embedded.'); + }); + + it('emits a prompt_use event keyed by the run root span (obs_id)', async () => { + const daemon = await start(); + vi.stubEnv('LOUSHO_OBSERVE', base(daemon)); + const { prompt } = await publishPrompt(base(daemon), { name: 'mira', text: 'registry text' }); + const model = mockModel(['ok']); + + const agent = createAgent({ name: 'prompt-agent-b', provider: model, prompt: 'mira@1' }); + await agent.send('hi'); + await closeReporter('prompt-agent-b', base(daemon)); + + const uses = daemon.db.prepare('SELECT * FROM prompt_uses').all(); + expect(uses).toHaveLength(1); + expect(String(uses[0].prompt_id)).toBe(prompt.prompt_id); + expect(Number(uses[0].version)).toBe(1); + const root = daemon.db.prepare('SELECT obs_id FROM observations WHERE parent_obs_id IS NULL').get(); + expect(root).toBeDefined(); + expect(String(uses[0].obs_id)).toBe(String(root?.obs_id)); + }); + + it('opts in via LOUSHO_PROMPT when config.prompt is unset', async () => { + const daemon = await start(); + vi.stubEnv('LOUSHO_OBSERVE', base(daemon)); + vi.stubEnv('LOUSHO_PROMPT', 'mira@latest'); + await publishPrompt(base(daemon), { name: 'mira', text: 'env fetched text' }); + const model = mockModel(['ok']); + + const agent = createAgent({ name: 'prompt-agent-c', provider: model }); + await agent.send('hi'); + await closeReporter('prompt-agent-c', base(daemon)); + + expect(systemPrompt(model)).toContain('env fetched text'); + }); +}); + +describe('createAgent({ prompt }) - fallback + conflicts', () => { + it('uses the embedded instructions and warns once when the daemon is unreachable', async () => { + vi.stubEnv('LOUSHO_OBSERVE', 'http://127.0.0.1:1'); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const model = mockModel(['ok']); + + const agent = createAgent({ name: 'prompt-agent-d', provider: model, prompt: 'mira@1', instructions: 'You are embedded.' }); + await agent.send('hi'); + await closeReporter('prompt-agent-d', 'http://127.0.0.1:1'); + + expect(systemPrompt(model)).toContain('You are embedded.'); + expect(warn.mock.calls.filter((c) => String(c[0]).includes('prompt'))).toHaveLength(1); + }); + + it('falls back to the default instructions when a ref is set but nothing is embedded', async () => { + vi.stubEnv('LOUSHO_OBSERVE', 'http://127.0.0.1:1'); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const model = mockModel(['ok']); + await createAgent({ name: 'prompt-agent-e', provider: model, prompt: 'mira@1' }).send('hi'); + await closeReporter('prompt-agent-e', 'http://127.0.0.1:1'); + expect(systemPrompt(model)).toContain('You are a helpful assistant.'); + }); + + it('still treats a non-ref prompt + instructions as the conflicting-aliases error', () => { + // @ts-expect-error - the union forbids a non-ref `prompt` beside `instructions`; the runtime guard still covers JS callers. + expect(() => createAgent({ provider: mockModel(['ok']), instructions: 'a', prompt: 'plain text prompt' })).toThrow( + /both 'instructions' and 'prompt'/ + ); + }); + + it('a non-ref prompt stays the deprecated instructions alias', async () => { + const model = mockModel(['ok']); + await createAgent({ name: 'prompt-agent-f', provider: model, prompt: 'Legacy alias text.' }).send('hi'); + expect(systemPrompt(model)).toContain('Legacy alias text.'); + }); +}); diff --git a/src/deploy/adapters/cloudflare.test.ts b/src/deploy/adapters/cloudflare.test.ts index 3036dd64..6af8e859 100644 --- a/src/deploy/adapters/cloudflare.test.ts +++ b/src/deploy/adapters/cloudflare.test.ts @@ -160,12 +160,13 @@ describe('CloudflareWorkerAdapter', () => { expect(findNodeBuiltinReferences('const x = "no builtins here"; import { x } from "./path";')).toEqual([]); }); - it('findNodeBuiltinReferences accepts only the allowlisted getBuiltinModule probes of ai v6/v7', () => { - const probes = 'loadBuiltinModule2("node:diagnostics_channel"); loadBuiltinModule4("node:dns"); process.getBuiltinModule?.("node:async_hooks"); loadBuiltinModule("node:module"); await loadNodeModule("node:dns");'; + it('findNodeBuiltinReferences accepts only the allowlisted getBuiltinModule probes', () => { + const probes = + 'loadBuiltinModule2("node:diagnostics_channel"); loadBuiltinModule4("node:dns"); process.getBuiltinModule?.("node:async_hooks"); loadBuiltinModule("node:module"); await loadNodeModule("node:dns"); getBuiltinModule("node:fs");'; expect(findNodeBuiltinReferences(probes)).toEqual([]); // Any other builtin, or an allowlisted id outside a probe call, is still a leak. - expect(findNodeBuiltinReferences('loadBuiltinModule("node:fs"); const id = "node:dns"; import("node:async_hooks");')).toEqual([ - '"node:fs"', + expect(findNodeBuiltinReferences('loadBuiltinModule("node:net"); const id = "node:dns"; import("node:async_hooks");')).toEqual([ + '"node:net"', '"node:dns"', '"node:async_hooks"', ]); diff --git a/src/deploy/adapters/cloudflare.ts b/src/deploy/adapters/cloudflare.ts index 8b7ff758..63134471 100644 --- a/src/deploy/adapters/cloudflare.ts +++ b/src/deploy/adapters/cloudflare.ts @@ -321,10 +321,14 @@ export default { fetch, scheduled }; * - node:diagnostics_channel, node:async_hooks: `ai`'s telemetry tracing * channel, used only when `process.release.name === 'node'`, and a missing * module is treated as "no subscribers". + * - node:fs, node:crypto, node:os, node:path: src/observe/identity.ts reaches + * the worker graph via createAgent's LOUSHO_OBSERVE wiring; every access is + * guarded by `typeof process === 'undefined'` and degrades to an + * in-memory reporter identity on runtimes without builtins. * Only these ids, and only as the argument of a `getBuiltinModule` / * `loadBuiltinModule` / `loadNodeModule` call, are exempt; any other `node:` string still fails. */ -const WORKER_SAFE_BUILTIN_PROBE = /\b(?:get|load)(?:Node|Builtin)Module\d*(?:\?\.)?\(\s*(["'`])node:(?:module|dns|diagnostics_channel|async_hooks)\1\s*\)/g; +const WORKER_SAFE_BUILTIN_PROBE = /\b(?:get|load)(?:Node|Builtin)Module\d*(?:\?\.)?\(\s*(["'`])node:(?:module|dns|diagnostics_channel|async_hooks|fs|crypto|os|path)\1\s*\)/g; /** Returns every `node:`-prefixed module specifier referenced in `source`. */ export function findNodeBuiltinReferences(source: string): string[] { diff --git a/src/deploy/bundle.ts b/src/deploy/bundle.ts index 6bca1135..b2b39e0d 100644 --- a/src/deploy/bundle.ts +++ b/src/deploy/bundle.ts @@ -91,13 +91,16 @@ export function sdkRuntimePlugin({ sdkEntry = 'main' }: { sdkEntry?: 'main' | 'w * esbuild plugin (cloudflare-worker target only) that redirects the SDK's * internal `security/sandboxCore` imports to the Worker-safe shim in * ./shims/sandboxCore.worker.ts - the real module's NoopSandbox needs - * node:child_process, which Workers don't have. And the optional peer - * `quickjs-emscripten` (code mode) to ./shims/quickjs.worker.ts, so the - * Emscripten runtime never lands in a Worker bundle. + * node:child_process, which Workers don't have. And the optional peers + * `quickjs-emscripten` (code mode) and `isolated-vm` (the 'isolate' code + * sandbox driver) to ./shims/quickjs.worker.ts and + * ./shims/isolatedVm.worker.ts, so neither the Emscripten runtime nor the + * native addon lands in a Worker bundle. */ export function workerSandboxShimPlugin(): Plugin { const shim = path.join(findSdkRoot(), 'src', 'deploy', 'shims', 'sandboxCore.worker.ts'); const quickjs = path.join(findSdkRoot(), 'src', 'deploy', 'shims', 'quickjs.worker.ts'); + const isolatedVm = path.join(findSdkRoot(), 'src', 'deploy', 'shims', 'isolatedVm.worker.ts'); return { name: 'lousho-worker-sandbox-shim', setup(build) { @@ -106,6 +109,9 @@ export function workerSandboxShimPlugin(): Plugin { return { path: shim }; }); build.onResolve({ filter: /^quickjs-emscripten$/ }, () => ({ path: quickjs })); + // The 'isolate' sandbox driver's lazy `import('isolated-vm')` becomes + // the shim's coded failure (the native addon cannot run in a Worker). + build.onResolve({ filter: /^isolated-vm$/ }, () => ({ path: isolatedVm })); }, }; } diff --git a/src/deploy/evalModule.ts b/src/deploy/evalModule.ts index cbb64fcd..f17820f3 100644 --- a/src/deploy/evalModule.ts +++ b/src/deploy/evalModule.ts @@ -9,7 +9,7 @@ * Node-only (`lousho build`); nothing here reaches a Worker bundle. */ import { SDKError } from '../execution/errors'; -import { sdkRuntimePlugin, workerUnsupportedPeerPlugin } from './bundle'; +import { sdkRuntimePlugin, workerSandboxShimPlugin, workerUnsupportedPeerPlugin } from './bundle'; import { explainWorkerBuildError } from './workerSdkExports'; /** esbuild, lazily: like tsup, only `lousho build` needs it (it comes with tsup). */ @@ -49,7 +49,7 @@ export async function evalModule(file: string): Promise> platform: 'node', target: 'es2022', logLevel: 'silent', - plugins: [sdkRuntimePlugin({ sdkEntry: 'worker' }), workerUnsupportedPeerPlugin()], + plugins: [sdkRuntimePlugin({ sdkEntry: 'worker' }), workerSandboxShimPlugin(), workerUnsupportedPeerPlugin()], }) .catch((error: unknown) => { throw explainWorkerBuildError(error); diff --git a/src/deploy/shims/isolatedVm.worker.ts b/src/deploy/shims/isolatedVm.worker.ts new file mode 100644 index 00000000..8f571e89 --- /dev/null +++ b/src/deploy/shims/isolatedVm.worker.ts @@ -0,0 +1,21 @@ +/** + * Worker-safe stand-in for the optional peer `isolated-vm`. + * + * The cloudflare-worker adapter's build redirects the SDK's import of + * `isolated-vm` (the 'isolate' sandbox driver's V8 isolate) here, so a + * Worker bundle never carries the native addon, which cannot run outside a + * Node process anyway. A `sandbox({ driver: 'isolate' })` run - or a + * `defineTool({ sandbox })` call - fails on first use with this error. + * The Isolate export must exist for `new module.Isolate(...)` to fail + * cleanly rather than with a confusing "not a constructor". + */ +import { SDKError } from '../../execution/errors'; + +export class Isolate { + constructor() { + throw new SDKError( + "The 'isolate' sandbox driver is not supported on the cloudflare-worker target: isolated-vm is a Node-only native module that is not bundled into Workers. Pass another SandboxDriver, or run the tool without a sandbox.", + 'LOUSHO_DEPLOY_FAILED' + ); + } +} diff --git a/src/evals/defineEval.ts b/src/evals/defineEval.ts index bd98ff93..429cbc31 100644 --- a/src/evals/defineEval.ts +++ b/src/evals/defineEval.ts @@ -13,6 +13,8 @@ import type * as Vitest from 'vitest'; import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../execution/AgentExecutor'; import { withEvalCassettes } from './cassettes'; import { scoreAssertion, type EvalResult } from './evalResult'; +import { isJudgeScorer } from './llmJudge'; +import { evalCaseObservation } from './observe'; import { matchesTagFilter, recordEvalResult } from './recorder'; import { remoteTargetFromEnv } from './remoteTarget'; import { SDKError } from '../execution/errors'; @@ -282,10 +284,20 @@ function defineClassicEval(config: EvalConfig): void { const file = currentTestPath(expect); const evalResult = await runCase({ name, tags: tags ?? [] }, file, context, async () => { const started = Date.now(); + // The exercised run parents under the eval's own span, so its trace is + // the one the emitted score rows name (LOUSHO_OBSERVE / --observe only). + const observation = evalCaseObservation({ evalName: name, agent: executeFields.agent, parentSpanId: executeFields.parentSpanId }); const nested: NestedCall[] = []; - const result = await AgentExecutor.execute({ ...executeFields, onAgentEvent: (event) => collectNested(event, nested) }); + const result = await observation.run((parentSpanId) => + AgentExecutor.execute({ + ...executeFields, + parentSpanId: parentSpanId ?? executeFields.parentSpanId, + onAgentEvent: (event) => collectNested(event, nested), + }) + ); const assertion = scoreAssertion(await score(result), threshold); - return { + if (isJudgeScorer(score)) assertion.source = 'judge'; + const evalResult: EvalResult = { name, tags: tags ?? [], passed: assertion.passed, @@ -296,6 +308,8 @@ function defineClassicEval(config: EvalConfig): void { usage: result.usage, file, }; + observation.emit(evalResult); + return evalResult; }); expect(evalResult.assertions[0].score).toBeGreaterThanOrEqual(threshold); }, timeoutMs); diff --git a/src/evals/evalResult.ts b/src/evals/evalResult.ts index 7d154682..13697d2d 100644 --- a/src/evals/evalResult.ts +++ b/src/evals/evalResult.ts @@ -15,6 +15,12 @@ export interface AssertionResult { name: string; kind: AssertionKind; passed: boolean; + /** + * Where the verdict came from: `'judge'` when an llmJudge() call produced + * the scored value, `'eval'` (absent) for everything else. The observability + * platform's `score` ingest events carry it as their `source` field. + */ + source?: 'eval' | 'judge'; /** 1 or 0 for yes/no assertions, the measured value for scored ones (judge, atLeast). */ score: number; /** Minimum score for the assertion to pass, when it has one. */ diff --git a/src/evals/llmJudge.ts b/src/evals/llmJudge.ts index 9bd55ce3..5d0e98e8 100644 --- a/src/evals/llmJudge.ts +++ b/src/evals/llmJudge.ts @@ -206,6 +206,14 @@ export function parseJudgeCritique(rawText: string): LLMCritique { return reason === undefined ? { score, feedback } : { score, feedback, reason }; } +/** Scorers llmJudge() built: their assertions emit `source: 'judge'` score rows (observability evaluators). */ +const judgeScorers = new WeakSet<(result: ExecutionResult) => Promise>(); + +/** True when `fn` is a scorer llmJudge() returned (so evals can mark its assertion `source: 'judge'`). */ +export function isJudgeScorer(fn: unknown): fn is (result: ExecutionResult) => Promise { + return typeof fn === 'function' && judgeScorers.has(fn as (result: ExecutionResult) => Promise); +} + /** * Returns a defineEval()-compatible scorer that grades an ExecutionResult's * `.text` against `config.rubric` using an LLM judge, via @@ -213,7 +221,7 @@ export function parseJudgeCritique(rawText: string): LLMCritique { * which this codebase's LLMProvider interface does not have). */ export function llmJudge(config: LLMJudgeConfig): (result: ExecutionResult) => Promise { - return async (result: ExecutionResult): Promise => { + const scorer = async (result: ExecutionResult): Promise => { assertInsideJudgeRunner(config, 'llmJudge()'); const responseText = await generateJudgeResponse(config, buildGradePrompt(config.rubric, userInputOf(result), result.text ?? '')); @@ -221,6 +229,8 @@ export function llmJudge(config: LLMJudgeConfig): (result: ExecutionResult) => P const { score } = parseJudgeScore(responseText); return score; }; + judgeScorers.add(scorer); + return scorer; } /** diff --git a/src/evals/observe.test.ts b/src/evals/observe.test.ts new file mode 100644 index 00000000..944f7823 --- /dev/null +++ b/src/evals/observe.test.ts @@ -0,0 +1,203 @@ +/** + * Eval score emission (observability evaluators): `scoreEventsOf` maps one + * score row per AssertionResult (`/`, judge vs eval source, + * failure comment); `evalCaseObservation` runs the case inside an `eval` + * span when LOUSHO_OBSERVE is set - the span's id is the trace_id the score + * rows and the exercised run share - and stays a plain no-op when it is not. + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createAgent } from '../createAgent'; +import { mockModel } from '../testing'; +import type { IngestEvent } from '../observe/envelope'; +import type { EvalResult } from './evalResult'; +import { evalCaseObservation, evalObservationEnabled, scoreEventsOf, scoredAgentName } from './observe'; +import { runTrajectoryCase } from './trajectory'; +import { atLeast } from './checks'; +import { remoteTarget } from './remoteTarget'; +import { z } from 'zod'; +import { defineTool } from '../tools/defineTool'; + +const noopFetch = () => { + const calls: { url: string; batch: IngestEvent[] }[] = []; + const fetchFn = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(input), batch: (JSON.parse(String(init?.body)) as { batch: IngestEvent[] }).batch }); + return new Response('{}', { status: 202 }); + }); + return { fetchFn, calls }; +}; + +const scoreEvents = (calls: { batch: IngestEvent[] }[]) => + calls.flatMap((c) => c.batch).filter((e): e is Extract => e.type === 'score'); + +function result(over: Partial = {}): EvalResult { + return { + name: 'refund eval', + tags: [], + passed: true, + assertions: [ + { name: 'completed()', kind: 'gate', passed: true, score: 1, threshold: 1 }, + { name: 'tone', kind: 'soft', passed: false, score: 0.2, threshold: 0.7, message: 'soft check failed', source: 'judge' }, + ], + durationMs: 5, + steps: 1, + toolCalls: [], + ...over, + }; +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('scoreEventsOf', () => { + it('maps one score row per AssertionResult with the spec naming and source', () => { + const events = scoreEventsOf(result(), { traceId: 'trace-1', agentId: 'agent-1', now: () => 42, newId: () => 'fresh-id' }); + expect(events).toHaveLength(2); + expect(events[0]).toEqual({ + type: 'score', + data: { + score_id: 'fresh-id', + trace_id: 'trace-1', + name: 'refund eval/completed()', + value: 1, + source: 'eval', + agent_id: 'agent-1', + ts: 42, + }, + }); + expect(events[1].type === 'score' && events[1].data).toMatchObject({ + name: 'refund eval/tone', + value: 0.2, + source: 'judge', + comment: 'soft check failed', + }); + }); + + it('omits comment and agent_id when absent', () => { + const events = scoreEventsOf(result({ assertions: [{ name: 'a', kind: 'gate', passed: true, score: 1 }] }), { traceId: 't' }); + const data = (events[0] as Extract).data; + expect(data.comment).toBeUndefined(); + expect(data.agent_id).toBeUndefined(); + }); +}); + +describe('scoredAgentName', () => { + it('names a createAgent agent by its config name (default agent) and a config by .name', () => { + expect(scoredAgentName(createAgent({ name: 'named-eval-agent', provider: mockModel(['x']) }))).toBe('named-eval-agent'); + expect(scoredAgentName(createAgent({ provider: mockModel(['x']) }))).toBe('agent'); + expect(scoredAgentName({ name: 'config-agent' })).toBe('config-agent'); + }); + + it('is undefined for remote targets and nothing', () => { + expect(scoredAgentName(remoteTarget({ url: 'http://x.test' })())).toBeUndefined(); + expect(scoredAgentName(undefined)).toBeUndefined(); + }); +}); + +describe('evalCaseObservation', () => { + it('runs the case with no span and emits nothing when LOUSHO_OBSERVE is unset', async () => { + const { fetchFn, calls } = noopFetch(); + vi.stubEnv('LOUSHO_OBSERVE', '0'); + vi.stubGlobal('fetch', fetchFn); + const observation = evalCaseObservation({ evalName: 'e' }); + let seen: string | undefined | 'unset' = 'unset'; + await observation.run(async (parentSpanId) => { + seen = parentSpanId; + }); + expect(seen).toBeUndefined(); + observation.emit(result()); + await Promise.resolve(); + expect(calls).toHaveLength(0); + expect(evalObservationEnabled({ LOUSHO_OBSERVE: '0' })).toBe(false); + expect(evalObservationEnabled({ LOUSHO_OBSERVE: '1' })).toBe(true); + }); + + it('parents the exercised run under the eval span and posts the score rows with its id as trace_id', async () => { + const { fetchFn, calls } = noopFetch(); + vi.stubEnv('LOUSHO_OBSERVE', '1'); + vi.stubEnv('LOUSHO_OBSERVE_PORT', '1'); + vi.stubGlobal('fetch', fetchFn); + const observation = evalCaseObservation({ evalName: 'refund eval', agent: { name: 'eval-observe-scored' } }); + let spanId: string | undefined; + await observation.run(async (parentSpanId) => { + spanId = parentSpanId; + }); + expect(spanId).toEqual(expect.any(String)); + observation.emit(result()); + await vi.waitFor(() => expect(calls.length).toBeGreaterThan(0)); + const scores = scoreEvents(calls); + expect(scores).toHaveLength(2); + for (const event of scores) { + expect(event.data.trace_id).toBe(spanId); + expect(event.data.agent_id).toEqual(expect.any(String)); + } + expect(scores[0].data.name).toBe('refund eval/completed()'); + }); +}); + +describe('runTrajectoryCase emission (integration)', () => { + const lookup = defineTool({ + name: 'lookup_order', + description: 'Look up an order', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId }), + }); + + it('emits score rows bound to the exercised run trace, judge verdicts as source judge', async () => { + const { fetchFn, calls } = noopFetch(); + vi.stubEnv('LOUSHO_OBSERVE', '1'); + vi.stubEnv('LOUSHO_OBSERVE_PORT', '1'); + vi.stubGlobal('fetch', fetchFn); + const agent = createAgent({ name: 'eval-scored-agent', prompt: 'p', provider: mockModel(['done.']), tools: [lookup] }); + const judge = { provider: mockModel(['SCORE: 0.9']) }; + const res = await runTrajectoryCase( + { + name: 'scored eval', + agent, + judge, + test: async (t) => { + await t.send('hi'); + t.completed(); + t.soft('tone', await t.judge('be nice'), atLeast(0.7)); + }, + }, + {}, + 'case a', + undefined + ); + expect(res.passed).toBe(true); + expect(res.assertions.find((a) => a.name === 'tone')?.source).toBe('judge'); + await vi.waitFor(() => expect(scoreEvents(calls)).toHaveLength(2)); + const scores = scoreEvents(calls); + const traceIds = new Set(scores.map((e) => e.data.trace_id)); + expect(traceIds.size).toBe(1); + expect(scores.map((e) => e.data.name).sort()).toEqual(['scored eval/completed()', 'scored eval/tone']); + expect(scores.find((e) => e.data.name === 'scored eval/tone')?.data.source).toBe('judge'); + expect(scores.find((e) => e.data.name === 'scored eval/completed()')?.data.source).toBe('eval'); + expect(scores.every((e) => typeof e.data.agent_id === 'string' && e.data.agent_id.length > 0)).toBe(true); + }); + + it('emits nothing when LOUSHO_OBSERVE is off, and the case still runs', async () => { + const { fetchFn, calls } = noopFetch(); + vi.stubEnv('LOUSHO_OBSERVE', '0'); + vi.stubGlobal('fetch', fetchFn); + const res = await runTrajectoryCase( + { + name: 'quiet eval', + agent: createAgent({ provider: mockModel(['hi.']) }), + test: async (t) => { + await t.send('hi'); + t.completed(); + }, + }, + {}, + undefined, + undefined + ); + expect(res.passed).toBe(true); + await Promise.resolve(); + expect(calls).toHaveLength(0); + }); +}); diff --git a/src/evals/observe.ts b/src/evals/observe.ts new file mode 100644 index 00000000..b6e34c3c --- /dev/null +++ b/src/evals/observe.ts @@ -0,0 +1,137 @@ +/** + * Eval scores as observability data (spec: docs/specs/0001-observability-platform/0001-evaluators.md). + * + * When `LOUSHO_OBSERVE` is set (or `lousho eval --observe` set it inside the + * vitest worker), each eval case runs inside an `eval ` span reported + * to the same collector reporter the exercised agent uses, and the case's + * assertions POST as `score` ingest events. The span's id becomes the + * `trace_id` of both the score rows and the exercised run's spans, so a + * failing score reaches the run that produced it in one click - the eval + * harness's own judgement context sits in the same trace as the root span. + * + * Emission lives here rather than in recorder.ts because it needs the span, + * not just the result. Everything degrades silently: `LOUSHO_OBSERVE` unset + * means no span and no POSTs, and a dead daemon is counted as dropped, never + * thrown into the eval (see src/observe/emit.ts). + */ + +import { createAgentConfigOf, type SimpleAgent } from '../createAgent'; +import { withSpan } from '../execution/tracing'; +import { emitObserveEvents, observeTargetFromEnv } from '../observe/emit'; +import { observeExporterFromEnv } from '../observe/reporter'; +import type { IngestEvent } from '../observe/envelope'; +import { newId } from '../utils/id'; +import type { EvalResult } from './evalResult'; +import { isRemoteTarget } from './remoteTarget'; + +/** + * The name the scored agent reports under: its `createAgent({ name })` (or a + * config's own `name`), defaulting to `'agent'` like createAgent does. + * `undefined` when there is no scored agent to attribute - a remoteTarget() + * (whose id lives on the deployment) or no agent at all. + */ +export function scoredAgentName(agent: unknown): string | undefined { + if (agent === undefined || agent === null || isRemoteTarget(agent)) return undefined; + const config = typeof agent === 'object' || typeof agent === 'function' ? createAgentConfigOf(agent as SimpleAgent) : undefined; + if (config) return config.name || 'agent'; + const name = (agent as { name?: unknown }).name; + return typeof name === 'string' && name ? name : 'agent'; +} + +/** + * One `score` ingest event per `AssertionResult`, the spec's mapping: + * `name` is `/`, llmJudge verdicts land with + * `source: 'judge'`, everything else `'eval'`. `score_id` is fresh per + * emission - a re-run is a new row, not an upsert, so eval history keeps + * every attempt. + */ +export function scoreEventsOf( + result: EvalResult, + options: { traceId: string; agentId?: string; now?: () => number; newId?: () => string } +): IngestEvent[] { + const ts = (options.now ?? Date.now)(); + const id = options.newId ?? newId; + return result.assertions.map((assertion) => ({ + type: 'score' as const, + data: { + score_id: id(), + trace_id: options.traceId, + name: `${result.name}/${assertion.name}`, + value: assertion.score, + source: assertion.source ?? 'eval', + ...(assertion.message !== undefined ? { comment: assertion.message } : {}), + ...(options.agentId !== undefined ? { agent_id: options.agentId } : {}), + ts, + }, + })); +} + +/** Whether eval cases emit score events and run inside an `eval` span (the same env check the emit path applies). */ +export function evalObservationEnabled(env: NodeJS.ProcessEnv | undefined = typeof process !== 'undefined' ? process.env : undefined): boolean { + return observeTargetFromEnv(env) !== undefined; +} + +/** One eval case's observation: the wrapper span to parent exercised runs under, and the score emit. */ +export interface EvalCaseObservation { + /** + * Runs `work` inside the case's `eval ` span when observation is on + * (`work` gets the span id to pass as `parentSpanId` to the exercised run, + * putting it in the same trace); a plain `work(undefined)` call when off. + */ + run(work: (parentSpanId: string | undefined) => Promise): Promise; + /** POSTs the case's score events (fire-and-forget); a no-op when observation is off or the case recorded no assertions. */ + emit(result: EvalResult): void; +} + +/** + * The observation context of one eval case. When `LOUSHO_OBSERVE` resolves a + * target, `run` wraps the case in an `eval ` span on the scored + * agent's own env reporter (same memoized instance -> the run's spans nest + * into this trace); `emit` posts one `score` event per assertion with that + * span's id as `trace_id` and the scored agent's `agent_id`. + */ +export function evalCaseObservation(options: { + /** The eval's name (the score rows' `/...` prefix). */ + evalName: string; + /** The dataset case's label, recorded on the eval span. */ + caseLabel?: string; + /** The scored agent (a SimpleAgent, a `{ name }` config, or an EvalTarget); resolved for its reporter name + `agent_id`. */ + agent?: unknown; + /** A span to parent the eval span to (the classic eval's own `parentSpanId` passthrough). */ + parentSpanId?: string; + /** Env override (tests); default `process.env`. */ + env?: NodeJS.ProcessEnv; +}): EvalCaseObservation { + const env = options.env ?? (typeof process !== 'undefined' ? process.env : undefined); + const scoredName = scoredAgentName(options.agent); + // The scored agent's reporter is the memoized env reporter under the same + // name - reporting the eval span to it puts the span and the exercised + // run's spans in one trace (`parentSpanId` resolves through its map). + // Without a scored agent (remote target) the harness reports as 'eval'. + const reporter = observeTargetFromEnv(env) === undefined ? undefined : observeExporterFromEnv({ agentName: scoredName ?? 'eval', env }); + let traceId: string | undefined; + return { + async run(work: (parentSpanId: string | undefined) => Promise): Promise { + if (!reporter) return work(undefined); + return withSpan( + reporter, + `eval ${options.evalName}`, + { + 'lousho.eval.name': options.evalName, + ...(options.caseLabel !== undefined ? { 'lousho.eval.case': options.caseLabel } : {}), + }, + (span) => { + traceId = span.id; + return work(span.id); + }, + options.parentSpanId + ); + }, + emit(result: EvalResult): void { + if (!reporter || traceId === undefined || result.assertions.length === 0) return; + const events = scoreEventsOf(result, { traceId, ...(scoredName !== undefined ? { agentId: reporter.agentId } : {}) }); + // Fire-and-forget: emitObserveEvents drops-and-counts, it never throws into the eval. + void emitObserveEvents(events, { env }); + }, + }; +} diff --git a/src/evals/remoteTarget.ts b/src/evals/remoteTarget.ts index fb12f169..afbe247d 100644 --- a/src/evals/remoteTarget.ts +++ b/src/evals/remoteTarget.ts @@ -13,8 +13,13 @@ import { newId } from '../utils/id'; /** What an eval case needs from the thing under test: `send()`. A `createAgent()` agent has it, and so does a remote target. */ export interface EvalTarget { - /** `options.onEvent` receives the run's events; a target that cannot stream them (a remote one) ignores it. */ - send(input: AgentInput, options?: { onEvent?: (event: AgentEvent) => void }): Promise; + /** + * `options.onEvent` receives the run's events; a target that cannot stream + * them (a remote one) ignores it. `options.parentSpanId` nests the run's + * `invoke_agent` span under that span (see SendOptions.parentSpanId); a + * remote target emits no local spans and ignores it. + */ + send(input: AgentInput, options?: { onEvent?: (event: AgentEvent) => void; parentSpanId?: string }): Promise; } /** Environment variables `lousho eval --url` / `--token` hand to the vitest worker. */ @@ -73,12 +78,22 @@ function toResult(summary: SessionTurnSummary): RemoteExecutionResult { export function remoteTarget(options: RemoteTargetOptions): () => EvalTarget { return () => { const sessionId = `eval-${newId()}`; - return { + const target: EvalTarget = { send: async (input) => toResult(await runRemoteTurn(options, { sessionId, input, label: LABEL })), }; + remoteTargets.add(target); + return target; }; } +/** Targets remoteTarget() built (a deployed agent: its id lives on the other side, so its scores carry no agent_id). */ +const remoteTargets = new WeakSet(); + +/** True when `value` is a remoteTarget()-built target. */ +export function isRemoteTarget(value: unknown): value is EvalTarget { + return typeof value === 'object' && value !== null && remoteTargets.has(value as EvalTarget); +} + /** The target `lousho eval --url` asked for (through the worker's environment), if any. */ export function remoteTargetFromEnv(env: Record = process.env): (() => EvalTarget) | undefined { const url = env[REMOTE_URL_ENV]; diff --git a/src/evals/trajectory.ts b/src/evals/trajectory.ts index 0d591f40..7930853b 100644 --- a/src/evals/trajectory.ts +++ b/src/evals/trajectory.ts @@ -18,6 +18,7 @@ import type { AssertionKind, AssertionResult, EvalResult, EvalToolCall } from '. import { gateFailures } from './evalResult'; import { callsFor, describeCalled, diffArgs, isSubsequence, parseToolCalls } from './toolMatch'; import type { EvalTarget, RemoteExecutionResult } from './remoteTarget'; +import { evalCaseObservation, evalObservationEnabled } from './observe'; import { SDKError } from '../execution/errors'; /** The judge provider a trajectory eval grades with, set via `defineEval({ judge })`. */ @@ -161,6 +162,10 @@ class TrajectoryContext implements EvalTestContext { private readonly results: ExecutionResult[] = []; private readonly calls: EvalToolCall[] = []; private readonly recorded: AssertionResult[] = []; + /** Scores t.judge() produced, consumed by the next check/soft over the same value (it lands `source: 'judge'`). */ + private judgeScores: number[] = []; + /** The eval-case span's id when observing; every send() parents its run under it (one trace per case). */ + parentSpanId?: string; private agent: EvalTarget | undefined; /** @@ -191,7 +196,10 @@ class TrajectoryContext implements EvalTestContext { async send(message: AgentInput): Promise { this.agent ??= typeof this.agentSource === 'function' ? await this.agentSource() : this.agentSource; const nested: NestedCall[] = []; - const result = await this.agent.send(message, { onEvent: (event) => collectNested(event, nested) }); + const result = await this.agent.send(message, { + onEvent: (event) => collectNested(event, nested), + parentSpanId: this.parentSpanId, + }); this.results.push(result); this.calls.push(...mergeNested(result.toolCalls ?? [], nested)); return result; @@ -269,7 +277,10 @@ class TrajectoryContext implements EvalTestContext { if (!this.judgeConfig) throw new SDKError(NO_JUDGE_MESSAGE, 'LOUSHO_EVALS_INVALID'); if (!this.result) throw new SDKError('t.judge() grades the latest reply - call t.send() first.', 'LOUSHO_EVALS_INVALID', { appendHelp: false }); const grade = llmJudge({ ...this.judgeConfig, rubric, allowOutsideJudgeRunner: true }); - return grade(this.result); + const value = await grade(this.result); + // Remember the verdict: the check/soft the score flows into marks itself judge-sourced. + this.judgeScores.push(value); + return value; } /** Builds the case's result from everything recorded so far. */ @@ -303,7 +314,18 @@ class TrajectoryContext implements EvalTestContext { private scored(kind: AssertionKind, name: string, value: T, check: Check): void { const outcome = check.evaluate(value); const message = outcome.passed ? undefined : `${kind === 'soft' ? 'soft ' : ''}check '${name}' failed: ${outcome.message ?? 'check did not pass'}`; - this.recorded.push({ name, kind, passed: outcome.passed, score: outcome.score, threshold: outcome.threshold, message }); + // A t.judge() verdict asserted over its own score is a judge-sourced assertion. + const judged = typeof value === 'number' ? this.judgeScores.indexOf(value) : -1; + if (judged >= 0) this.judgeScores.splice(judged, 1); + this.recorded.push({ + name, + kind, + passed: outcome.passed, + score: outcome.score, + threshold: outcome.threshold, + message, + ...(judged >= 0 ? { source: 'judge' as const } : {}), + }); } private calledToolProblem(name: string, options: CalledToolOptions): string | undefined { @@ -340,15 +362,34 @@ export async function runTrajectoryCase( file: string | undefined ): Promise { const started = Date.now(); - const context = new TrajectoryContext(spec.agent, spec.judge); + // When observing, a factory agent resolves eagerly: the eval span reports to + // the resolved agent's own env reporter so the exercised run's spans nest + // into the case's trace. Swallow a throw - it resurfaces as the case's error. + let agentSource: AgentSource = spec.agent; + let exercised: SimpleAgent | EvalTarget | undefined; + if (evalObservationEnabled() && typeof agentSource === 'function') { + try { + exercised = await agentSource(); + agentSource = exercised; + } catch { + // the factory's error becomes the case's error at the first send() + } + } + const observation = evalCaseObservation({ evalName: spec.name, caseLabel: label, agent: exercised ?? agentSource }); + const context = new TrajectoryContext(agentSource, spec.judge); let error: string | undefined; try { - await spec.test(context, c); + await observation.run(async (parentSpanId) => { + context.parentSpanId = parentSpanId; + await spec.test(context, c); + }); } catch (caught) { error = caught instanceof Error ? caught.message : String(caught); } const base = { name: spec.name, case: label, tags: spec.tags ?? [], file }; - return context.toResult(base, Date.now() - started, error); + const result = context.toResult(base, Date.now() - started, error); + observation.emit(result); + return result; } /** The error message a failed case throws into vitest: every failed gate, one per line. */ diff --git a/src/execution/AgentExecutor.ts b/src/execution/AgentExecutor.ts index dca9385b..e41ff6fe 100644 --- a/src/execution/AgentExecutor.ts +++ b/src/execution/AgentExecutor.ts @@ -26,6 +26,7 @@ import type { CallUsage, RunUsage, StepUsage } from '../models/usage'; import { attachRunUsage, mergeDelegatedUsage, recordStepUsage } from './runUsage'; import { recordRunFailure } from './runFailure'; import { TraceExporter, withSpan } from './tracing'; +import { withObserveExporter } from '../observe/reporter'; import { agentRunSpanInit, recordToolOutcome, @@ -41,6 +42,7 @@ import { type ToolCallScope, } from './subagentRuntime'; import { ConfigurationError, isAbortError, isCassetteError, SDKError } from './errors'; +import { sandboxErrorKind } from '../sandbox/errors'; import { abortReasonOf } from './abortReason'; import type { HostedTool } from '../tools/hosted'; import { assertHostedToolsSupported, countHostedCalls, withHostedCalls } from './hostedToolCalls'; @@ -829,8 +831,14 @@ export class AgentExecutor { // throwing because `provider` was never checked). this.validateExecuteOptions(options); + // `LOUSHO_OBSERVE` fans the run's exporter out with the collector + // reporter (a no-op when unset); a delegated child already carries its + // lead's composed exporter, so it is not wired twice (src/observe/reporter.ts). + const exporter = withObserveExporter(options.exporter, options.agent?.name); + const run = exporter === options.exporter ? options : { ...options, exporter }; + // The run reports to its listeners through its event sink. - return observeRun(options, (observed) => { + return observeRun(run, (observed) => { // The entire run is wrapped in a top-level 'agent.run' span, // whose `id` is threaded as `parentId` into the nested 'llm.generate' // and 'tool.call' spans, giving the 3-level span tree its parent/child @@ -1764,22 +1772,40 @@ export class AgentExecutor { concurrency: options.toolConcurrency ?? 'unbounded', }); } - const executed = await this.executeToolCall( - toolCall, - agent, - toolRegistry, - onToolCall, - onToolResult, - sandbox, - hooks, - sessionId, - state.messages, - signal, - onPrepared, - // A delegated child's usage is added to this run's totals. - (child, errors) => mergeDelegatedUsage(state.usage, child, errors), - scope - ); + let executed: ToolCallOutcome; + try { + executed = await this.executeToolCall( + toolCall, + agent, + toolRegistry, + onToolCall, + onToolResult, + sandbox, + hooks, + sessionId, + state.messages, + signal, + onPrepared, + // A delegated child's usage is added to this run's totals. + (child, errors) => mergeDelegatedUsage(state.usage, child, errors), + scope + ); + } catch (error) { + // A call that throws (instead of settling an error outcome) still + // gets its boundary-failure kind on the span before it propagates. + recordToolOutcome( + toolSpan, + { + args: undefined, + result: undefined, + error: error instanceof Error ? error.message : String(error), + sandboxError: sandboxErrorKind(error), + latencyMs: Date.now() - toolCallStart, + }, + { redactContent, captureContent: resolveCaptureContent(options.captureContent) } + ); + throw error; + } const parsedArgs = executed.args === undefined ? parseToolArguments(toolCall, toolCall.function.arguments) @@ -1790,6 +1816,7 @@ export class AgentExecutor { args: parsedArgs, result: executed.result, error: executed.error, + sandboxError: executed.sandboxError, latencyMs: Date.now() - toolCallStart, }, { redactContent, captureContent: resolveCaptureContent(options.captureContent) } diff --git a/src/execution/codeMode.ts b/src/execution/codeMode.ts index dc4c2184..d2e0c17f 100644 --- a/src/execution/codeMode.ts +++ b/src/execution/codeMode.ts @@ -341,7 +341,7 @@ export function nestedToolCaller(setup: NestedCallSetup): NestedToolCaller { const settled = inScript(await runToolCall(toolCall, { ...base, signal, scope, ...(onToolPartial && { onToolPartial }) })); recordToolOutcome( span, - { args: settled.args ?? args, result: settled.result, error: settled.error, latencyMs: Date.now() - started }, + { args: settled.args ?? args, result: settled.result, error: settled.error, sandboxError: settled.sandboxError, latencyMs: Date.now() - started }, { redactContent: tracing.redactContent, captureContent: resolveCaptureContent(tracing.captureContent) } ); return settled; diff --git a/src/execution/genAiSpans.ts b/src/execution/genAiSpans.ts index 74752a2b..a7264395 100644 --- a/src/execution/genAiSpans.ts +++ b/src/execution/genAiSpans.ts @@ -17,7 +17,8 @@ import type { GenerateOptions, GenerateResult, LLMProvider, Message, ToolCall } import { textOf } from '../providers/content'; import type { ToolRegistry } from '../tools'; import type { AgentConfig } from '../types'; -import { ErrorAttr, GenAiAttr, GenAiOperation, LegacyAttr, SdkAttr, TOOL_TYPE_FUNCTION, CAPTURE_CONTENT_ENV } from './semconv'; +import { ErrorAttr, GenAiAttr, GenAiOperation, LegacyAttr, SandboxAttr, SdkAttr, TOOL_TYPE_FUNCTION, CAPTURE_CONTENT_ENV } from './semconv'; +import { sandboxDriverName } from '../sandbox/types'; import type { Span, SpanKind } from './tracing'; import { estimateCost } from '../models'; import { normalizeUsage } from '../models/usage'; @@ -264,6 +265,7 @@ export function toolSpanInit( context: { agent: AgentConfig; toolRegistry?: ToolRegistry; sessionId?: string } ): SpanInit { const toolName = toolCall.name; + const desc = context.toolRegistry?.get(toolName); return { name: `${GenAiOperation.EXECUTE_TOOL} ${toolName}`, kind: 'internal', @@ -271,11 +273,13 @@ export function toolSpanInit( [GenAiAttr.OPERATION_NAME]: GenAiOperation.EXECUTE_TOOL, [GenAiAttr.TOOL_NAME]: toolName, [GenAiAttr.TOOL_CALL_ID]: toolCall.id, - [GenAiAttr.TOOL_DESCRIPTION]: context.toolRegistry?.get(toolName)?.tool?.description, + [GenAiAttr.TOOL_DESCRIPTION]: desc?.tool?.description, [GenAiAttr.TOOL_TYPE]: TOOL_TYPE_FUNCTION, [GenAiAttr.AGENT_NAME]: context.agent.name, [GenAiAttr.CONVERSATION_ID]: context.sessionId, [LegacyAttr.TOOL_NAME]: toolName, + // A `defineTool({ sandbox })` tool: which driver ran it (`'isolate'` by default). + [SandboxAttr.DRIVER]: desc?.sandbox !== undefined ? sandboxDriverName(desc.sandbox.driver) : undefined, }), }; } @@ -283,7 +287,7 @@ export function toolSpanInit( /** Records args/result (content), error and latency on an `execute_tool` span. */ export function recordToolOutcome( span: Span, - outcome: { args: unknown; result: unknown; error?: string; latencyMs: number }, + outcome: { args: unknown; result: unknown; error?: string; sandboxError?: string; latencyMs: number }, content: ContentOptions & { captureContent: boolean } ): void { span.attributes = { @@ -292,6 +296,8 @@ export function recordToolOutcome( ...(!keepLegacyContent(content) ? {} : { [LegacyAttr.ARGS]: outcome.args, [LegacyAttr.RESULT]: outcome.result }), // The message, as on every other span's `error`; absent when the tool succeeded. [LegacyAttr.ERROR]: outcome.error, + // The sandbox boundary's failure kind ('timeout' | 'oom' | 'driver'), absent for the tool's own errors. + [SandboxAttr.ERROR]: outcome.sandboxError, [LegacyAttr.LATENCY_MS]: outcome.latencyMs, [ErrorAttr.TYPE]: outcome.error ? 'tool_error' : undefined, [GenAiAttr.TOOL_CALL_ARGUMENTS]: content.captureContent ? JSON.stringify(outcome.args) : undefined, diff --git a/src/execution/resume.ts b/src/execution/resume.ts index 7107ac3e..405c8642 100644 --- a/src/execution/resume.ts +++ b/src/execution/resume.ts @@ -55,6 +55,7 @@ import type { OAuthTokenStore } from '../oauth/types'; import { isSignInRequired, settleSignInRequired, signInOwner, signInRequest, SignInPendingError, type SignInRequired } from '../oauth/signIn'; import { newId } from '../utils/id'; import { agentRunSpanInit, recordToolOutcome, resolveCaptureContent, toolSpanInit, SUBAGENT_SPAN, type SubagentSpanInfo } from './genAiSpans'; +import { sandboxErrorKind } from '../sandbox/errors'; import { withSpan, type Span } from './tracing'; /** @@ -1103,10 +1104,22 @@ async function runApprovedInSpan(ctx: ResumeContext, pending: PendingApproval, s try { const message = await run(); const { result, error } = toolResultOf(message); - recordToolOutcome(span, { args: pending.args, result, error, latencyMs: Date.now() - startedAt }, { redactContent, captureContent: resolveCaptureContent(executeOptions.captureContent) }); + // The structured error the model got carries the boundary failure kind, when it was one. + const sandboxError = (result as { sandboxError?: unknown } | null | undefined)?.sandboxError; + recordToolOutcome( + span, + { args: pending.args, result, error, sandboxError: typeof sandboxError === 'string' ? sandboxError : undefined, latencyMs: Date.now() - startedAt }, + { redactContent, captureContent: resolveCaptureContent(executeOptions.captureContent) } + ); return { message }; } catch (error) { if (isSignInRequired(error)) return { signIn: error }; + // A thrown failure still gets its boundary-failure kind before it propagates. + recordToolOutcome( + span, + { args: pending.args, result: undefined, error: error instanceof Error ? error.message : String(error), sandboxError: sandboxErrorKind(error), latencyMs: Date.now() - startedAt }, + { redactContent, captureContent: resolveCaptureContent(executeOptions.captureContent) } + ); throw error; } }, diff --git a/src/execution/semconv.ts b/src/execution/semconv.ts index b6c69dd1..186053d6 100644 --- a/src/execution/semconv.ts +++ b/src/execution/semconv.ts @@ -94,6 +94,18 @@ export const ErrorAttr = { /** `gen_ai.tool.type` value for tools executed by this SDK's host process. */ export const TOOL_TYPE_FUNCTION = 'function'; +/** + * SDK-specific `sandbox.*` attributes of `execute_tool` spans for tools that + * run inside a code sandbox (`defineTool({ sandbox })`, spec + * 0001-sandboxing). No GenAI convention exists for these yet. + */ +export const SandboxAttr = { + /** The driver that ran the call (`'isolate'`, a custom driver's `name`, ...); set only on a sandboxed tool's span. */ + DRIVER: 'sandbox.driver', + /** Why the sandbox boundary itself failed: `'timeout'`, `'oom'` or `'driver'`; absent when the tool's own code failed. */ + ERROR: 'sandbox.error', +} as const; + /** Attributes specific to this SDK (not part of the GenAI semantic conventions). */ export const SdkAttr = { /** `true` on a span whose token counts are (partly) estimates because the provider reported none. */ diff --git a/src/execution/toolCallExecution.ts b/src/execution/toolCallExecution.ts index 1a45331d..ec918742 100644 --- a/src/execution/toolCallExecution.ts +++ b/src/execution/toolCallExecution.ts @@ -69,6 +69,13 @@ export interface ToolCallOutcome { * `approvalTtlMs` applies. */ approvalTtlMs?: number; + /** + * Set when the call failed at a code-sandbox boundary + * (`defineTool({ sandbox })`): `'timeout'`, `'oom'` or `'driver'`. It + * becomes the span's `sandbox.error` attribute; the model's error result + * carries it as `result.sandboxError`. + */ + sandboxError?: string; } /** Everything a tool call needs from the surrounding execute() run. */ @@ -553,7 +560,12 @@ function thrownToolFailure(toolCall: ToolCall, error: unknown, ctx: ToolCallCont { toolName: toolCall.function.name, toolCallId: toolCall.id, error, plainMessage: toolErrorMessage(error) }, ctx.scope?.runtime.toolErrorMessage ); - return { ...toolFailure(toolCall, 'execution', message), result }; + return { + ...toolFailure(toolCall, 'execution', message), + result, + // A sandbox boundary failure's kind reaches the span as `sandbox.error`. + ...(result.sandboxError !== undefined && { sandboxError: result.sandboxError }), + }; } /** diff --git a/src/execution/toolErrors.ts b/src/execution/toolErrors.ts index a714b0a7..cbd14fba 100644 --- a/src/execution/toolErrors.ts +++ b/src/execution/toolErrors.ts @@ -5,6 +5,8 @@ * failed sandbox guard) hands the model the object {@link toolErrorResult} builds. */ +import { sandboxErrorKind } from '../sandbox/errors'; + /** Why a tool call produced an error result. */ export type ToolErrorKind = | 'execution' @@ -53,6 +55,12 @@ export interface ToolErrorResult { kind: ToolErrorKind; /** The model's id for the call, when the caller supplied it. */ toolCallId?: string; + /** + * Set when a `kind: 'sandbox'` failure happened at the sandbox boundary + * (`defineTool({ sandbox })`): `'timeout'`, `'oom'` or `'driver'` - the + * `sandbox.error` span attribute's value. + */ + sandboxError?: string; /** Kind-specific extras, such as `issues` (validation) or `note` (rejected). */ [extra: string]: unknown; } @@ -90,8 +98,11 @@ export function toolErrorResult(input: ToolErrorInput): ToolErrorResult { const kind = input.kind ?? declaredKind(error); const err = typeof error === 'string' ? undefined : (error as { name?: unknown; message?: unknown } | null | undefined); const raw = typeof error === 'string' ? error : typeof err?.message === 'string' ? err.message : String(error); + // A SandboxError's kind ('timeout'|'oom'|'driver') rides along so the span can carry `sandbox.error`. + const sandboxError = sandboxErrorKind(error); return { ...details, + ...(sandboxError !== undefined && { sandboxError }), error: typeof err?.name === 'string' && err.name ? err.name : DEFAULT_ERROR_NAME[kind], toolName, message: diff --git a/src/execution/tracing.ts b/src/execution/tracing.ts index 2f9585ed..6e19fbb3 100644 --- a/src/execution/tracing.ts +++ b/src/execution/tracing.ts @@ -166,7 +166,9 @@ function applyRollup(span: Span, own: UsageRollup): void { * tracing backend they use (console, OpenTelemetry, Datadog, ...). */ export interface TraceExporter { + /** A span opened; the exporter maps it onto its trace (parents by `span.parentId`). */ onSpanStart(span: Span): void; + /** A span finished (status and end time set) - the finished-observation point for reporters. */ onSpanEnd(span: Span): void; } diff --git a/src/flows/FlowExecutor.ts b/src/flows/FlowExecutor.ts index 9ce04eb3..3b2374a1 100644 --- a/src/flows/FlowExecutor.ts +++ b/src/flows/FlowExecutor.ts @@ -37,6 +37,7 @@ import { toolSpanInit, } from '../execution/genAiSpans'; import { FLOW_NODE_SPAN_NAME, FlowAttr, GenAiAttr, GenAiOperation } from '../execution/semconv'; +import { sandboxErrorKind } from '../sandbox/errors'; import { CompactedLLMProviderError, ConfigurationError, @@ -1368,16 +1369,30 @@ export class FlowExecutor { async (toolSpan) => { const start = Date.now(); // The tool gets the run's signal as `ctx.abortSignal`, and the gate's call id. - const result = await executeToolWithSandboxGuard(toolName, toolDesc, args, sandbox, context.signal, { toolCallId }); - recordToolOutcome( - toolSpan, - { args, result, latencyMs: Date.now() - start }, - { - redactContent: context.redactContent, - captureContent: resolveCaptureContent(context.captureContent), - } - ); - return result; + try { + const result = await executeToolWithSandboxGuard(toolName, toolDesc, args, sandbox, context.signal, { toolCallId }); + recordToolOutcome( + toolSpan, + { args, result, latencyMs: Date.now() - start }, + { + redactContent: context.redactContent, + captureContent: resolveCaptureContent(context.captureContent), + } + ); + return result; + } catch (error) { + // A thrown failure (a code-sandbox boundary failure included) is + // recorded with its `sandbox.error` kind before it propagates. + recordToolOutcome( + toolSpan, + { args, result: undefined, error: error instanceof Error ? error.message : String(error), sandboxError: sandboxErrorKind(error), latencyMs: Date.now() - start }, + { + redactContent: context.redactContent, + captureContent: resolveCaptureContent(context.captureContent), + } + ); + throw error; + } }, context.parentSpanId, init.kind diff --git a/src/index.ts b/src/index.ts index fa33e175..6722bf73 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,6 +33,12 @@ export * from './evals'; // Security export * from './security'; +// Code sandboxing: sandbox({ driver }) runs code isolated ('isolate' via the +// optional isolated-vm peer, custom SandboxDriver, reserved 'docker'/'e2b'); +// defineTool({ sandbox }) routes a tool's execute through it. An execution +// boundary, not a security boundary - see docs/specs/0001-observability-platform/0001-sandboxing.md. +export * from './sandbox'; + // Storage moved to '@lousho/build-ai-agent/utils'. @@ -117,3 +123,8 @@ export * from './server/uiMessageStream'; // Route handler for Next.js and other Fetch-API frameworks. export * from './server/routeHandler'; + +// Context Hub client (spec 0001 observability platform): `hub()` builds the +// shared-memory client when LOUSHO_OBSERVE names a collector; `hubTools()` +// adapts it into hub_publish/hub_query agent tools. +export { hub, hubTools, HUB_SCOPES, type Hub, type HubOptions, type HubEntry, type HubScope, type HubPublishInput, type HubQuery } from './observe/hubClient'; diff --git a/src/observe/alerts.ts b/src/observe/alerts.ts new file mode 100644 index 00000000..45354396 --- /dev/null +++ b/src/observe/alerts.ts @@ -0,0 +1,159 @@ +/** + * Monitoring alert hooks: fire-and-forget JSON POSTs to webhook URLs. + * + * Sources of config, merged by {@link resolveAlertConfig}: + * - repeatable `--webhook ` on `lousho observe start`, and + * - `alerts.json` beside the db: `{"webhooks": [...], "errorRateThreshold": n}` + * (n is a fraction 0..1 of today's runs that errored). + * + * The sweep (sweep.ts) produces {@link AlertEvent}s; {@link createAlerter} + * delivers them. Delivery is deliberately dumb: plain `fetch`, a timeout so a + * hung endpoint cannot pin the event loop, warn-once per webhook on failure, + * and never a throw - alerting must not block ingest or the sweep (same rule + * the reporter lives by: telemetry failure never fails a run). + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { SDKError } from '../execution/errors'; + +/** The alerts config file, read from the directory holding the observe db. */ +const ALERTS_FILENAME = 'alerts.json'; + +/** A webhook call is abandoned after this; the sweep must not wait on a hung endpoint. */ +const ALERT_TIMEOUT_MS = 5000; + +/** One alert, POSTed verbatim as the JSON body to every configured webhook. */ +export interface AlertEvent { + /** 'agent.status' for a transition to 'down'; 'agent.error_rate' for a threshold crossing. */ + type: 'agent.status' | 'agent.error_rate'; + agent_id: string; + project: string | null; + /** 'down' for status alerts; the agent's current status for error-rate alerts. */ + status: string; + /** Epoch milliseconds when the sweep fired it. */ + at: number; + /** Alert-type-specific context (from-status, interrupted counts, rate vs threshold). */ + details: Record; +} + +export interface AlertConfig { + webhooks: string[]; + /** Today's error rate at which an `agent.error_rate` alert fires (0..1); absent disables that alert. */ + errorRateThreshold?: number; +} + +/** `alerts.json`'s path for a given db path (`observe.db` -> sibling `alerts.json`). */ +function alertsPathFor(dbPath: string): string { + return path.join(path.dirname(path.resolve(dbPath)), ALERTS_FILENAME); +} + +/** + * Read and validate an `alerts.json`. A missing file is just "no config"; + * a present-but-malformed file is a config error worth failing loudly on - + * silently ignoring it would leave the owner thinking alerts are on. + */ +export function loadAlertsFile(filePath: string): AlertConfig { + let text: string; + try { + text = fs.readFileSync(filePath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { webhooks: [] }; + throw new SDKError(`Could not read the alerts config at ${filePath}: ${(error as Error).message}`, 'LOUSHO_CONFIG_INVALID', { + appendHelp: false, + }); + } + let parsed: unknown; + try { + parsed = JSON.parse(text) as unknown; + } catch { + throw new SDKError( + `The alerts config at ${filePath} is not valid JSON (expected {"webhooks": ["", ...], "errorRateThreshold": 0.5}).`, + 'LOUSHO_CONFIG_INVALID', + { appendHelp: false } + ); + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new SDKError( + `The alerts config at ${filePath} must be a JSON object: {"webhooks": ["", ...], "errorRateThreshold": 0.5}.`, + 'LOUSHO_CONFIG_INVALID', + { appendHelp: false } + ); + } + const { webhooks: rawWebhooks, errorRateThreshold } = parsed as { webhooks?: unknown; errorRateThreshold?: unknown }; + const webhooks = rawWebhooks ?? []; + if (!Array.isArray(webhooks) || webhooks.some((w) => typeof w !== 'string' || w.trim() === '')) { + throw new SDKError(`The alerts config at ${filePath}: "webhooks" must be an array of URL strings.`, 'LOUSHO_CONFIG_INVALID', { + appendHelp: false, + }); + } + if ( + errorRateThreshold !== undefined && + (typeof errorRateThreshold !== 'number' || !Number.isFinite(errorRateThreshold) || errorRateThreshold < 0 || errorRateThreshold > 1) + ) { + throw new SDKError( + `The alerts config at ${filePath}: "errorRateThreshold" must be a number between 0 and 1 (a fraction of today's runs).`, + 'LOUSHO_CONFIG_INVALID', + { appendHelp: false } + ); + } + return { + webhooks: webhooks as string[], + ...(errorRateThreshold !== undefined ? { errorRateThreshold } : {}), + }; +} + +/** + * The daemon's alert config: `alerts.json` beside the db merged with + * `--webhook` flags (deduped). An in-memory db has no sibling file. + */ +export function resolveAlertConfig(dbPath: string, extraWebhooks: readonly string[] = []): AlertConfig { + const file = dbPath === ':memory:' ? { webhooks: [] as string[] } : loadAlertsFile(alertsPathFor(dbPath)); + return { + webhooks: [...new Set([...file.webhooks, ...extraWebhooks])], + ...(file.errorRateThreshold !== undefined ? { errorRateThreshold: file.errorRateThreshold } : {}), + }; +} + +export interface Alerter { + /** POST every alert to every webhook. Fire-and-forget: never throws, never awaited by callers. */ + post(alerts: readonly AlertEvent[]): void; +} + +/** + * Fire-and-forget webhook delivery. A webhook that errors (network or + * non-2xx) warns ONCE - a fleet going down at 3am must not fill the daemon's + * log with one line per agent per sweep, and the webhook may simply be down. + */ +export function createAlerter( + config: AlertConfig, + fetchFn: typeof fetch = fetch, + warn: (message: string) => void = (message) => console.warn(message) +): Alerter { + const warned = new Set(); + const warnOnce = (url: string, what: string): void => { + if (warned.has(url)) return; + warned.add(url); + warn(`[lousho observe] alert webhook ${url} ${what}; alerts keep firing, this warning is only logged once.`); + }; + return { + post(alerts) { + for (const url of config.webhooks) { + for (const alert of alerts) { + void fetchFn(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(alert), + signal: AbortSignal.timeout(ALERT_TIMEOUT_MS), + }) + .then((res) => { + if (!res.ok) warnOnce(url, `answered ${res.status}`); + }) + .catch((error: unknown) => { + warnOnce(url, `failed: ${error instanceof Error ? error.message : String(error)}`); + }); + } + } + }, + }; +} diff --git a/src/observe/daemon.test.ts b/src/observe/daemon.test.ts new file mode 100644 index 00000000..e6f0cd97 --- /dev/null +++ b/src/observe/daemon.test.ts @@ -0,0 +1,172 @@ +/** + * End-to-end daemon tests over real HTTP on an ephemeral port: the health + * marker, ingest 202 + truncation + 413, the traces/insights/agents/status + * read endpoints, bearer auth when required, and the "already running" + * port-collision detection. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { isObserveDaemon, probeHealth, startObserveDaemon, type ObserveDaemon } from './daemon'; +import { OBSERVE_SERVICE } from './envelope'; +import { createToken } from './store'; + +const T0 = 1_700_000_000_000; + +const daemons: ObserveDaemon[] = []; +afterEach(async () => { + while (daemons.length > 0) await daemons.pop()?.close(); +}); + +async function start(opts: Parameters[0] = {}): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0, ...opts }); + daemons.push(daemon); + return daemon; +} + +const url = (daemon: ObserveDaemon, path: string): string => `http://127.0.0.1:${daemon.port}${path}`; + +const obs = (over: Record = {}): Record => ({ + obs_id: 'o1', + trace_id: 't1', + session_id: 's1', + agent_id: 'a1', + kind: 'span', + name: 'invoke_agent a', + start: T0, + end: T0 + 100, + status: 'ok', + ...over, +}); + +const post = (daemon: ObserveDaemon, body: unknown, headers: Record = {}): Promise => + fetch(url(daemon, '/api/v1/ingest'), { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); + +describe('GET /health', () => { + it('returns the lousho-observe service marker', async () => { + const daemon = await start(); + const res = await fetch(url(daemon, '/health')); + const body = (await res.json()) as { service: string; ok: boolean; version: string }; + expect(res.status).toBe(200); + expect(body.service).toBe(OBSERVE_SERVICE); + expect(body.ok).toBe(true); + expect(typeof body.version).toBe('string'); + expect(isObserveDaemon(await probeHealth(daemon.port))).toBe(true); + }); +}); + +describe('POST /api/v1/ingest', () => { + it('accepts a batch and serves it back through the read endpoints', async () => { + const daemon = await start(); + const res = await post(daemon, { + batch: [ + { type: 'agent.heartbeat', data: { agent_id: 'a1', project: 'p', name: 'a' } }, + { type: 'observation', data: obs() }, + { type: 'observation', data: obs({ obs_id: 'o2', parent_obs_id: 'o1', kind: 'tool', name: 'call_tool x' }) }, + { type: 'score', data: { score_id: 's1', trace_id: 't1', name: 'eval/acc', value: 0.9, source: 'judge' } }, + ], + }); + expect(res.status).toBe(202); + + const list = (await (await fetch(url(daemon, '/api/v1/traces'))).json()) as { traces: { trace_id: string }[]; total: number }; + expect(list.total).toBe(1); + expect(list.traces[0].trace_id).toBe('t1'); + + const detail = (await (await fetch(url(daemon, '/api/v1/traces/t1'))).json()) as { + observations: { obs_id: string; parent_obs_id: string | null }[]; + scores: { name: string }[]; + }; + // flat rows; the panel nests by parent_obs_id + expect(detail.observations[0].obs_id).toBe('o1'); + expect(detail.observations[1].obs_id).toBe('o2'); + expect(detail.observations[1].parent_obs_id).toBe('o1'); + expect(detail.scores[0].name).toBe('eval/acc'); + + const scores = (await (await fetch(url(daemon, '/api/v1/scores'))).json()) as { scores: { name: string; value: number }[] }; + expect(scores.scores[0].name).toBe('eval/acc'); + expect(scores.scores[0].value).toBe(0.9); + + const agents = (await (await fetch(url(daemon, '/api/v1/agents'))).json()) as { agents: { agent_id: string }[] }; + expect(agents.agents[0].agent_id).toBe('a1'); + }); + + it('rejects a malformed envelope with 400', async () => { + const daemon = await start(); + const res = await post(daemon, { batch: [{ type: 'nope', data: {} }] }); + expect(res.status).toBe(400); + expect((await res.json()) as { code: string }).toMatchObject({ code: 'LOUSHO_REQUEST_INVALID' }); + }); + + it('rejects a body over the size cap with 413', async () => { + const daemon = await start(); + const res = await post(daemon, { + batch: [{ type: 'observation', data: obs({ input: 'x'.repeat(6 * 1024 * 1024) }) }], + }); + expect(res.status).toBe(413); + }); + + it('truncates a field over 64KB and flags the stored row', async () => { + const daemon = await start(); + const big = 'y'.repeat(70 * 1024); + await post(daemon, { batch: [{ type: 'observation', data: obs({ output: big }) }] }); + const detail = (await (await fetch(url(daemon, '/api/v1/traces/t1'))).json()) as { + observations: { output?: string; truncated: boolean }[]; + }; + expect(detail.observations[0].output?.length).toBe(64 * 1024); + expect(detail.observations[0].truncated).toBe(true); + }); +}); + +describe('status + insights', () => { + it('reports counts through /api/v1/status and aggregates through /api/v1/insights', async () => { + const daemon = await start(); + await post(daemon, { + batch: [ + { type: 'agent.heartbeat', data: { agent_id: 'a1', project: 'p' } }, + { type: 'observation', data: obs({ cost_usd: 0.01, usage_input_tokens: 10, usage_output_tokens: 5 }) }, + ], + }); + const status = (await (await fetch(url(daemon, '/api/v1/status'))).json()) as { + service: string; + counts: { agents: number; observations: number; traces: number }; + }; + expect(status.service).toBe(OBSERVE_SERVICE); + expect(status.counts).toMatchObject({ agents: 1, observations: 1, traces: 1 }); + + // the default window is "last 7 days" and the fixture timestamps are fixed - bound it explicitly + // (bucket_start must fall inside [since, until), so reach back a full bucket) + const insights = (await ( + await fetch(url(daemon, `/api/v1/insights?granularity=hour&since=${T0 - 3_600_000}&until=${T0 + 86_400_000}`)) + ).json()) as { + agents: { agent_id: string }[]; + series: { metric: string; value: number }[]; + }; + expect(insights.agents[0]?.agent_id).toBe('a1'); + const runs = insights.series.find((r) => r.metric === 'runs'); + expect(runs?.value).toBe(1); + }); +}); + +describe('token auth', () => { + it('rejects /api/v1/* without a bearer token when requireToken is set, but /health stays open', async () => { + const daemon = await start({ requireToken: true }); + expect((await fetch(url(daemon, '/api/v1/traces'))).status).toBe(401); + expect((await fetch(url(daemon, '/api/v1/ingest'), { method: 'POST', body: '{}' })).status).toBe(401); + expect((await fetch(url(daemon, '/health'))).status).toBe(200); + + const { key } = createToken(daemon.db, 'proj'); + const authed = await fetch(url(daemon, '/api/v1/traces'), { headers: { authorization: `Bearer ${key}` } }); + expect(authed.status).toBe(200); + }); +}); + +describe('port collision', () => { + it('a second daemon on the same port fails with EADDRINUSE; the health probe identifies ours', async () => { + const daemon = await start(); + await expect(startObserveDaemon({ dbPath: ':memory:', port: daemon.port })).rejects.toMatchObject({ code: 'EADDRINUSE' }); + // the probe distinguishes our daemon from a random listener + expect(isObserveDaemon(await probeHealth(daemon.port))).toBe(true); + }); +}); diff --git a/src/observe/daemon.ts b/src/observe/daemon.ts new file mode 100644 index 00000000..794c8722 --- /dev/null +++ b/src/observe/daemon.ts @@ -0,0 +1,272 @@ +/** + * The collector daemon: a loopback-only `node:http` server owning the only + * handle to the observe database (single-writer rule; Studio reads through + * its routes, never the file). + * + * - Bind address defaults to `127.0.0.1`: the threat model trusts any + * loopback process. A non-loopback `--host` turns token auth ON by itself + * (the node server rule: no unauthenticated public bind), and + * `--require-token` turns it on explicitly. `/health` always answers + * without auth - it is the `lousho-observe` marker the port-collision + * probe needs. + * - Routes live in `src/observe/routes/*` and dispatch through + * `matchRoute()`; the daemon owns body limits, auth, and error mapping, + * never per-endpoint logic. + * - On `EADDRINUSE` the CLI probes `GET /health`: a `lousho-observe` answer + * means "already running" (exit 0); anything else is a real collision. + */ + +import * as http from 'node:http'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import { SDKError } from '../execution/errors'; +import { isLoopbackHost } from '../server/hostCheck'; +import type { SqlDatabase } from '../storage/sqlite/driver'; +import { OBSERVE_DEFAULT_PORT, OBSERVE_SERVICE } from './envelope'; +import { BodyTooLargeError, matchRoute, sendJson, type RouteContext } from './routes/index'; +import { createAlerter, resolveAlertConfig, type AlertConfig } from './alerts'; +import { startStatusSweep, type StatusSweep } from './sweep'; +import { openDb, verifyToken } from './store'; + +/** `~/.lousho/observe.db`. */ +export function defaultDbPath(homeDir: string = os.homedir()): string { + return path.join(homeDir, '.lousho', 'observe.db'); +} + +/** Where the pidfile goes for a given db path (same directory, fixed name). */ +export function defaultPidPath(dbPath: string): string { + return path.join(path.dirname(resolveForFs(dbPath)), 'observe.pid'); +} + +function resolveForFs(p: string): string { + return p === ':memory:' ? p : path.resolve(p); +} + +export interface ObserveDaemonOptions { + /** SQLite file; default `~/.lousho/observe.db` (`:memory:` for tests). */ + dbPath?: string; + /** Default 7777; `0` picks an ephemeral port (tests). */ + port?: number; + /** Bind address; default `127.0.0.1`. Non-loopback forces token auth on. */ + host?: string; + /** Require `Authorization: Bearer ` on `/api/v1/*` even on loopback. */ + requireToken?: boolean; + /** Write the pid here on listen, remove on close (for `lousho observe stop`). */ + pidPath?: string; + /** An already-open db (tests inject one); otherwise `openDb(dbPath)`. */ + db?: SqlDatabase; + /** Injectable for tests; defaults to the global fetch. Used by the alert webhooks. */ + fetchFn?: typeof fetch; + /** Extra alert webhooks (repeatable `--webhook`); merged with `alerts.json` beside the db. */ + webhooks?: string[]; + /** Status-sweep cadence in ms (default 30s); `<= 0` leaves the sweep manual-only (tests). */ + sweepIntervalMs?: number; + /** Injectable clock for the sweep (tests); default `Date.now`. */ + nowFn?: () => number; +} + +export interface ObserveDaemon { + port: number; + host: string; + dbPath: string; + /** Whether bearer auth is enforced (non-loopback bind implies it). */ + requireToken: boolean; + server: http.Server; + db: SqlDatabase; + pidPath?: string; + /** The resolved alert config (`alerts.json` + `--webhook` flags). */ + alerts: AlertConfig; + /** The running status sweep (`run()` forces a pass; `stop()` is called by `close()`). */ + sweep: StatusSweep; + close(): Promise; +} + +function readBodyCapped(req: http.IncomingMessage, limitBytes: number): Promise { + return new Promise((resolvePromise, rejectPromise) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + let overflowed = false; + req.on('data', (chunk: Buffer) => { + if (settled && !overflowed) return; + if (overflowed) return; // drain the rest so the 413 response can land cleanly + size += chunk.length; + if (size > limitBytes) { + // Keep draining (discarding) - answering mid-upload and closing the socket + // makes clients report "socket closed" instead of seeing the 413. + overflowed = true; + chunks.length = 0; + return; + } + chunks.push(chunk); + }); + req.on('end', () => { + if (settled) return; + settled = true; + if (overflowed) { + rejectPromise(new BodyTooLargeError(`Request body exceeds the ${Math.round(limitBytes / 1024 / 1024)}MB limit.`)); + return; + } + resolvePromise(Buffer.concat(chunks).toString('utf8')); + }); + req.on('error', (error) => { + if (!settled) { + settled = true; + rejectPromise(error); + } + }); + }); +} + +/** The `Authorization: Bearer ` value, or undefined. */ +function bearerOf(req: http.IncomingMessage): string | undefined { + const header = req.headers.authorization; + const match = /^Bearer\s+(\S+)\s*$/i.exec(header ?? ''); + return match?.[1]; +} + +function isApiPath(pathname: string): boolean { + return pathname.startsWith('/api/'); +} + +/** Start the daemon. Rejects with the listen error (`EADDRINUSE` keeps `code` for the CLI's health probe). */ +export async function startObserveDaemon(options: ObserveDaemonOptions = {}): Promise { + const dbPath = options.dbPath ?? defaultDbPath(); + const host = options.host ?? '127.0.0.1'; + const port = options.port ?? OBSERVE_DEFAULT_PORT; + // Resolve before opening anything: a malformed alerts.json should fail the + // start loudly (silently dropping it would leave alerts "configured" but dead). + const alertConfig = resolveAlertConfig(dbPath, options.webhooks ?? []); + const db = options.db ?? openDb(dbPath); + const requireToken = options.requireToken === true || !isLoopbackHost(host); + + const onRequest = async (req: http.IncomingMessage, res: http.ServerResponse): Promise => { + const { pathname, searchParams } = new URL(req.url ?? '/', 'http://localhost'); + if (requireToken && isApiPath(pathname) && !verifyToken(db, bearerOf(req))) { + return sendJson(res, 401, { + error: 'This observe daemon requires a bearer token; create one with `lousho observe token create --project `.', + code: 'LOUSHO_REMOTE_UNAUTHORIZED', + }); + } + const match = matchRoute(req.method ?? 'GET', pathname); + if (!match) return sendJson(res, 404, { error: 'not found', code: 'LOUSHO_REQUEST_INVALID' }); + let bodyRead: Promise | undefined; + const ctx: RouteContext = { + db, + req, + res, + params: match.params, + query: searchParams, + readBody(limitBytes: number) { + bodyRead ??= readBodyCapped(req, limitBytes); + return bodyRead; + }, + }; + try { + await match.route.handle(ctx); + } catch (error) { + if (error instanceof BodyTooLargeError) { + return sendJson(res, 413, { error: error.message, code: 'LOUSHO_REQUEST_INVALID' }); + } + if (error instanceof SDKError && error.code === 'LOUSHO_REQUEST_INVALID') { + return sendJson(res, 400, { error: error.detail, code: error.code }); + } + console.error('[lousho observe] request failed:', error); + if (!res.headersSent) sendJson(res, 500, { error: 'internal error', code: 'LOUSHO_GENERIC_ERROR' }); + else res.end(); + } + }; + + const server = http.createServer((req, res) => { + void onRequest(req, res); + }); + + try { + await new Promise((resolvePromise, rejectPromise) => { + server.once('error', rejectPromise); + server.once('listening', () => { + server.removeListener('error', rejectPromise); + resolvePromise(); + }); + server.listen(port, host); + }); + } catch (error) { + // A failed bind (EADDRINUSE and friends) must not leak the db handle the + // daemon opened above - the CLI exits on the error, but tests and embedded + // callers would keep the WAL lock forever. + if (!options.db) db.close(); + throw error; + } + + const address = server.address(); + const boundPort = typeof address === 'object' && address !== null ? address.port : port; + + const pidPath = options.pidPath; + if (pidPath !== undefined) { + fs.mkdirSync(path.dirname(pidPath), { recursive: true }); + fs.writeFileSync(pidPath, String(process.pid), 'utf8'); + } + + // The monitoring sweep: rewrites agents.status from heartbeat freshness, + // interrupts a crashed agent's open work, and posts alerts to webhooks + // (fire-and-forget; delivery lives in alerts.ts). + const alerter = createAlerter(alertConfig, options.fetchFn ?? fetch); + const sweep = startStatusSweep(db, { + intervalMs: options.sweepIntervalMs, + now: options.nowFn, + errorRateThreshold: alertConfig.errorRateThreshold, + onAlerts: (alerts) => alerter.post(alerts), + }); + + return { + port: boundPort, + host, + dbPath, + requireToken, + server, + db, + alerts: alertConfig, + sweep, + ...(pidPath !== undefined ? { pidPath } : {}), + close: async () => { + sweep.stop(); + if (pidPath !== undefined) { + try { + fs.rmSync(pidPath, { force: true }); + } catch { + // best effort; a stale pidfile is reported by `stop`, not fatal + } + } + await new Promise((resolvePromise, rejectPromise) => + server.close((error) => (error ? rejectPromise(error) : resolvePromise())) + ); + if (!options.db) db.close(); + }, + }; +} + +export interface HealthProbe { + service: string; + ok: boolean; + version?: string; +} + +/** + * `GET /health` on `host:port`; `undefined` when nothing answers. Used by + * `start`'s EADDRINUSE handling and by `lousho observe status`. + */ +export async function probeHealth(port: number, host = '127.0.0.1', fetchFn: typeof fetch = fetch): Promise { + try { + const res = await fetchFn(`http://${host}:${port}/health`, { signal: AbortSignal.timeout(2000) }); + if (!res.ok) return undefined; + return (await res.json()) as HealthProbe; + } catch { + return undefined; + } +} + +/** Whether the health answer is a lousho-observe daemon's (the attach-detection marker). */ +export function isObserveDaemon(health: HealthProbe | undefined): health is HealthProbe { + return health?.service === OBSERVE_SERVICE && health.ok === true; +} diff --git a/src/observe/emit.test.ts b/src/observe/emit.test.ts new file mode 100644 index 00000000..8c64e259 --- /dev/null +++ b/src/observe/emit.test.ts @@ -0,0 +1,84 @@ +/** + * The event post path for non-span emitters: `LOUSHO_OBSERVE` resolves the + * target exactly like the reporter's env wiring, a batch POSTs once, and a + * dead daemon drops-and-counts (warning once) instead of throwing. + */ +import { describe, expect, it, vi } from 'vitest'; +import type { IngestEvent } from './envelope'; +import { emitObserveEvent, emitObserveEvents, observeTargetFromEnv } from './emit'; + +const score = (id: string): IngestEvent => ({ + type: 'score', + data: { score_id: id, trace_id: 't1', name: 'eval/acc', value: 1, source: 'eval' }, +}); + +function mockFetch(status = 202): { fetchFn: typeof fetch; calls: { url: string; body: { batch: IngestEvent[] }; headers: Record }[] } { + const calls: { url: string; body: { batch: IngestEvent[] }; headers: Record }[] = []; + const fetchFn = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + calls.push({ + url: String(input), + body: JSON.parse(String(init?.body)) as { batch: IngestEvent[] }, + headers: (init?.headers ?? {}) as Record, + }); + return new Response('{}', { status }); + }); + return { fetchFn: fetchFn as unknown as typeof fetch, calls }; +} + +describe('observeTargetFromEnv', () => { + it('is undefined when LOUSHO_OBSERVE is unset, 0 or false', () => { + expect(observeTargetFromEnv({})).toBeUndefined(); + expect(observeTargetFromEnv({ LOUSHO_OBSERVE: '0' })).toBeUndefined(); + expect(observeTargetFromEnv({ LOUSHO_OBSERVE: 'false' })).toBeUndefined(); + expect(observeTargetFromEnv(undefined)).toBeUndefined(); + }); + + it('maps 1/true to the loopback default (with LOUSHO_OBSERVE_PORT) and a URL through', () => { + expect(observeTargetFromEnv({ LOUSHO_OBSERVE: '1' })?.url).toBe('http://127.0.0.1:7777'); + expect(observeTargetFromEnv({ LOUSHO_OBSERVE: 'true', LOUSHO_OBSERVE_PORT: '8899' })?.url).toBe('http://127.0.0.1:8899'); + expect(observeTargetFromEnv({ LOUSHO_OBSERVE: 'http://collector.test:1234/' })?.url).toBe('http://collector.test:1234'); + }); + + it('carries the token and project', () => { + const target = observeTargetFromEnv({ LOUSHO_OBSERVE: '1', LOUSHO_OBSERVE_TOKEN: 'lso_x', LOUSHO_OBSERVE_PROJECT: 'p' }); + expect(target).toMatchObject({ token: 'lso_x', project: 'p' }); + }); +}); + +describe('emitObserveEvents', () => { + it('posts the batch to /api/v1/ingest and resolves the delivered count', async () => { + const { fetchFn, calls } = mockFetch(); + const delivered = await emitObserveEvents([score('s1'), score('s2')], { env: { LOUSHO_OBSERVE: 'http://d.test:9' }, fetchFn }); + expect(delivered).toBe(2); + expect(calls).toHaveLength(1); + expect(calls[0].url).toBe('http://d.test:9/api/v1/ingest'); + expect(calls[0].body.batch.map((e) => e.type)).toEqual(['score', 'score']); + }); + + it('sends the bearer token when the target has one', async () => { + const { fetchFn, calls } = mockFetch(); + await emitObserveEvent(score('s1'), { env: { LOUSHO_OBSERVE: '1', LOUSHO_OBSERVE_TOKEN: 'lso_abc' }, fetchFn }); + expect(calls[0].headers.authorization).toBe('Bearer lso_abc'); + }); + + it('does nothing (and calls no fetch) when LOUSHO_OBSERVE is unset or off', async () => { + const { fetchFn, calls } = mockFetch(); + expect(await emitObserveEvents([score('s1')], { env: {}, fetchFn })).toBe(0); + expect(await emitObserveEvents([score('s1')], { env: { LOUSHO_OBSERVE: '0' }, fetchFn })).toBe(0); + expect(calls).toHaveLength(0); + }); + + it('drops and counts on a non-2xx answer or a transport failure, warning once, never throwing', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + try { + const failing = mockFetch(500); + expect(await emitObserveEvents([score('s1'), score('s2')], { env: { LOUSHO_OBSERVE: '1' }, fetchFn: failing.fetchFn })).toBe(0); + const throwing = { fetchFn: vi.fn(async () => Promise.reject(new Error('conn refused'))) as unknown as typeof fetch }; + expect(await emitObserveEvents([score('s3')], { env: { LOUSHO_OBSERVE: '1' }, fetchFn: throwing.fetchFn })).toBe(0); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0][0])).toContain('dropped'); + } finally { + warn.mockRestore(); + } + }); +}); diff --git a/src/observe/emit.ts b/src/observe/emit.ts new file mode 100644 index 00000000..602e91f8 --- /dev/null +++ b/src/observe/emit.ts @@ -0,0 +1,105 @@ +/** + * `emitObserveEvents()` - the event post path for emitters that are not the + * span reporter: eval scores land when a case finishes, not when a span ends, + * so `src/evals` needs a way to POST ingest events on its own cadence. + * + * The rules mirror the reporter's (src/observe/reporter.ts) because the + * contract is the same: + * - `LOUSHO_OBSERVE` decides where events go: `1`/`true` -> the loopback + * default, a URL -> that collector, unset/`0`/`false` -> nothing is sent + * ({@link observeTargetFromEnv} returns `undefined`). + * - One `POST /api/v1/ingest` per call, `{ batch: [...] }` as the envelope + * declares; the daemon's ingest is the only write path. + * - Drop, never throw: an unreachable daemon or a non-2xx answer warns ONCE + * and resolves 0. An eval run, a human feedback widget, anything calling + * this must never fail because telemetry did. + * - The POST is plain `fetch` outside any span, like the reporter's own. + */ + +import { SDKError } from '../execution/errors'; +import { OBSERVE_DEFAULT_PORT, type IngestEvent } from './envelope'; + +/** Where events go: the collector's base URL plus its optional bearer token. */ +export interface ObserveTarget { + url: string; + token?: string; + /** The project label the reporter stamps; emitters resolving agent identity want the same value. */ + project?: string; +} + +const DEFAULT_TIMEOUT_MS = 5000; + +/** + * The collector `LOUSHO_OBSERVE` points at, with the same resolution the + * reporter applies (`observeExporterFromEnv`): `1`/`true` -> the loopback + * daemon (`LOUSHO_OBSERVE_PORT`, default 7777), a URL -> that collector, + * unset/`0`/`false` -> `undefined` (emission is off). + * + * `lousho eval --observe` sets `LOUSHO_OBSERVE=1` inside the vitest worker + * when the variable is unset, so this one check covers both the env-driven + * and the forced case. + */ +export function observeTargetFromEnv(env: NodeJS.ProcessEnv | undefined = typeof process !== 'undefined' ? process.env : undefined): ObserveTarget | undefined { + const flag = env?.LOUSHO_OBSERVE?.trim(); + if (!env || !flag || flag === '0' || flag.toLowerCase() === 'false') return undefined; + const url = + flag === '1' || flag.toLowerCase() === 'true' + ? `http://127.0.0.1:${env.LOUSHO_OBSERVE_PORT || String(OBSERVE_DEFAULT_PORT)}` + : flag; + return { + url: url.replace(/\/+$/, ''), + ...(env.LOUSHO_OBSERVE_TOKEN ? { token: env.LOUSHO_OBSERVE_TOKEN } : {}), + ...(env.LOUSHO_OBSERVE_PROJECT ? { project: env.LOUSHO_OBSERVE_PROJECT } : {}), + }; +} + +export interface EmitObserveOptions { + /** Explicit target; default the env's (`observeTargetFromEnv`). */ + target?: ObserveTarget; + /** Env to resolve the target from when `target` is absent (tests). */ + env?: NodeJS.ProcessEnv; + /** Injectable transport (tests); default the global `fetch`. */ + fetchFn?: typeof fetch; + /** Per-request timeout (default 5000ms). */ + timeoutMs?: number; +} + +let warned = false; + +/** + * POST `events` as one ingest batch and resolve with the count delivered + * (0 when emission is off or the POST failed). Never throws: failures warn + * once, like the reporter. + */ +export async function emitObserveEvents(events: readonly IngestEvent[], options: EmitObserveOptions = {}): Promise { + const target = options.target ?? observeTargetFromEnv(options.env); + if (target === undefined || events.length === 0) return 0; + const headers: Record = { 'content-type': 'application/json' }; + if (target.token) headers.authorization = `Bearer ${target.token}`; + try { + const res = await (options.fetchFn ?? fetch)(`${target.url}/api/v1/ingest`, { + method: 'POST', + headers, + body: JSON.stringify({ batch: [...events] }), + signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS), + }); + if (!res.ok) { + throw new SDKError(`ingest answered ${res.status}`, 'LOUSHO_REMOTE_REQUEST_FAILED', { appendHelp: false }); + } + return events.length; + } catch (error) { + if (!warned) { + warned = true; + const reason = error instanceof Error ? error.message : String(error); + console.warn( + `[lousho] observe: could not reach the observe daemon at ${target.url} (${reason}); ${events.length} event(s) dropped and the run is unaffected.` + ); + } + return 0; + } +} + +/** One-event convenience over {@link emitObserveEvents}. */ +export function emitObserveEvent(event: IngestEvent, options: EmitObserveOptions = {}): Promise { + return emitObserveEvents([event], options); +} diff --git a/src/observe/envelope.test.ts b/src/observe/envelope.test.ts new file mode 100644 index 00000000..7f290146 --- /dev/null +++ b/src/observe/envelope.test.ts @@ -0,0 +1,94 @@ +/** The ingest envelope is the single contract shared by reporter and daemon; these pin its acceptance and rejection edges. */ +import { describe, expect, it } from 'vitest'; +import { parseIngestEnvelope, OBSERVE_MAX_BATCH_EVENTS } from './envelope'; + +const observation = { + obs_id: 'obs-1', + trace_id: 'trace-1', + kind: 'span', + name: 'invoke_agent a', + start: 1000, + end: 1100, +}; + +describe('parseIngestEnvelope', () => { + it('accepts a well-formed batch of every event type', () => { + const result = parseIngestEnvelope({ + batch: [ + { type: 'agent.heartbeat', data: { agent_id: 'a1', project: 'p' } }, + { type: 'agent.deregister', data: { agent_id: 'a1' } }, + { type: 'observation', data: observation }, + { type: 'prompt_use', data: { obs_id: 'obs-1', prompt_id: 'p1', version: 2 } }, + { type: 'score', data: { score_id: 's1', trace_id: 'trace-1', name: 'eval/x', value: 1, source: 'judge' } }, + ], + }); + expect(result.ok).toBe(true); + }); + + it('accepts an empty batch', () => { + expect(parseIngestEnvelope({ batch: [] }).ok).toBe(true); + }); + + it('rejects a body that is not an envelope', () => { + for (const body of [undefined, null, {}, { batch: {} }, { batch: [1] }, '[]']) { + expect(parseIngestEnvelope(body).ok, JSON.stringify(body)).toBe(false); + } + }); + + it('rejects an unknown event type', () => { + const result = parseIngestEnvelope({ batch: [{ type: 'hub.entry', data: {} }] }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.issues.join(' ')).toContain('batch'); + }); + + it('rejects an observation without obs_id/trace_id/kind', () => { + for (const data of [{ trace_id: 't', kind: 'span', start: 1 }, { obs_id: 'o', kind: 'span', start: 1 }, { obs_id: 'o', trace_id: 't', start: 1 }, { obs_id: 'o', trace_id: 't', kind: 'span' }]) { + const result = parseIngestEnvelope({ batch: [{ type: 'observation', data }] }); + expect(result.ok, JSON.stringify(data)).toBe(false); + } + }); + + it('rejects a bad observation kind and a bad score source', () => { + expect(parseIngestEnvelope({ batch: [{ type: 'observation', data: { ...observation, kind: 'trace' } }] }).ok).toBe(false); + expect( + parseIngestEnvelope({ batch: [{ type: 'score', data: { score_id: 's', trace_id: 't', name: 'n', source: 'auto' } }] }).ok + ).toBe(false); + }); + + it('rejects a heartbeat without agent_id', () => { + expect(parseIngestEnvelope({ batch: [{ type: 'agent.heartbeat', data: { project: 'p' } }] }).ok).toBe(false); + }); + + it('rejects a batch over the event cap', () => { + const batch = Array.from({ length: OBSERVE_MAX_BATCH_EVENTS + 1 }, (_, i) => ({ + type: 'agent.heartbeat' as const, + data: { agent_id: `a${i}` }, + })); + expect(parseIngestEnvelope({ batch }).ok).toBe(false); + }); + + it('keeps optional usage and cost fields', () => { + const result = parseIngestEnvelope({ + batch: [ + { + type: 'observation', + data: { + ...observation, + usage_input_tokens: 10, + usage_output_tokens: 5, + usage_cached_tokens: 2, + usage_reasoning_tokens: 1, + cost_usd: 0.001, + cost_partial: true, + usage_estimated: true, + }, + }, + ], + }); + expect(result.ok).toBe(true); + if (result.ok) { + const event = result.envelope.batch[0]; + expect(event.type === 'observation' && event.data.cost_partial).toBe(true); + } + }); +}); diff --git a/src/observe/envelope.ts b/src/observe/envelope.ts new file mode 100644 index 00000000..a67ac173 --- /dev/null +++ b/src/observe/envelope.ts @@ -0,0 +1,153 @@ +/** + * The ingest envelope: the one contract between reporters (any process that + * emits agent telemetry) and the collector daemon, and the shape a hosted + * collector implements later (see docs/specs/0001-observability-platform). + * + * `POST /api/v1/ingest` takes `{ batch: [{ type, data }] }`. Every event type + * the platform knows is declared here once - reporter and daemon share these + * schemas as the single source of truth, so a field rename cannot silently + * drift between emit and store. + * + * The event types cover later children too (`score` for evaluators, + * `prompt_use` for the prompt registry, `agent.heartbeat`/`agent.deregister` + * for monitoring): the data plane accepts and stores them all even where its + * own read surface does not serve them yet. + */ + +import { z } from 'zod'; +import { issueMessage, issuePath } from '../utils/zodCompat'; + +/** The daemon's default port; `LOUSHO_OBSERVE=1` points reporters at it. */ +export const OBSERVE_DEFAULT_PORT = 7777; + +/** The `service` value `GET /health` answers; `lousho observe start` probes it to tell "already running" from a real port collision. */ +export const OBSERVE_SERVICE = 'lousho-observe'; + +/** Request bodies larger than this get a `413`; a reporter counts the batch into `dropped`. */ +export const OBSERVE_MAX_BODY_BYTES = 5 * 1024 * 1024; + +/** `input`/`output` strings longer than this are cut and the row's `truncated` flag set (the flag must survive: Studio shows it). */ +export const OBSERVE_TRUNCATE_CHARS = 64 * 1024; + +/** One ingest call carries at most this many events; bounds the write transaction's size. */ +export const OBSERVE_MAX_BATCH_EVENTS = 1000; + +/** The `kind` values of an observation row (the wide table's one discriminator). */ +const OBSERVATION_KINDS = ['span', 'generation', 'tool', 'event'] as const; +export type ObservationKind = (typeof OBSERVATION_KINDS)[number]; + +/** A score's origin: an eval assertion, an llmJudge verdict, or human feedback. */ +export const SCORE_SOURCES = ['eval', 'judge', 'human'] as const; + +/** `agent.heartbeat` - one row of the fleet registry, refreshed on the reporter's interval. */ +const agentHeartbeatData = z.object({ + agent_id: z.string().min(1), + project: z.string().optional(), + name: z.string().optional(), + version: z.string().optional(), + /** Milliseconds since the epoch; the daemon fills in receive time when absent. */ + ts: z.number().optional(), + /** The reporter's heartbeat cadence, so monitoring can derive freshness without guessing. */ + heartbeat_interval_ms: z.number().optional(), + hostname: z.string().optional(), + pid: z.number().optional(), + meta: z.record(z.string(), z.unknown()).optional(), +}); +/** `agent.deregister` - a graceful exit; the registry row flips to `stopped` instead of aging into `down`. */ +const agentDeregisterData = z.object({ + agent_id: z.string().min(1), + ts: z.number().optional(), + reason: z.string().optional(), +}); +/** + * `observation` - one finished span (`obs_id` is the SDK's `TraceLine.id`, so a + * resent batch upserts by key and `updated_at` gives last-writer-wins). The + * `usage_*`/`cost_*` fields carry `StepUsage` verbatim; the daemon stores them + * and never recomputes - `cost_partial` and `usage_estimated` must reach + * Studio untouched or cost numbers lie. + */ +const observationData = z.object({ + obs_id: z.string().min(1), + trace_id: z.string().min(1), + session_id: z.string().optional(), + agent_id: z.string().optional(), + parent_obs_id: z.string().optional(), + kind: z.enum(OBSERVATION_KINDS), + name: z.string().optional(), + /** Milliseconds since the epoch. */ + start: z.number(), + end: z.number().optional(), + status: z.enum(['ok', 'error']).optional(), + input: z.string().optional(), + output: z.string().optional(), + model: z.string().optional(), + usage_input_tokens: z.number().nonnegative().optional(), + usage_output_tokens: z.number().nonnegative().optional(), + usage_cached_tokens: z.number().nonnegative().optional(), + usage_reasoning_tokens: z.number().nonnegative().optional(), + cost_usd: z.number().nonnegative().optional(), + cost_partial: z.boolean().optional(), + usage_estimated: z.boolean().optional(), + truncated: z.boolean().optional(), + /** The span's full attribute map (`gen_ai.*` names), kept verbatim as JSON. */ + attrs: z.record(z.string(), z.unknown()).optional(), +}); +export type ObservationData = z.infer; + +/** `prompt_use` - links a run's observation to the registry prompt version it ran with. */ +export interface PromptUseData { + obs_id: string; + prompt_id: string; + version?: number; +} +// Explicit annotation: the DTS rollup emits this const next to the public +// type, and zod 3's expanded ZodObject generics are not valid under zod 4. +const promptUseData: z.ZodType = z.object({ + obs_id: z.string().min(1), + prompt_id: z.string().min(1), + version: z.number().int().optional(), +}); + +/** `score` - one eval/judge/human verdict attached to a trace (and optionally one observation in it). */ +const scoreData = z.object({ + score_id: z.string().min(1), + trace_id: z.string().min(1), + obs_id: z.string().optional(), + name: z.string().min(1), + value: z.number().optional(), + source: z.enum(SCORE_SOURCES), + comment: z.string().optional(), + /** Which agent the score is about; the emitting eval harness may differ from the scored agent. */ + agent_id: z.string().optional(), + ts: z.number().optional(), +}); +/** One event of the batch. The `type` discriminator decides which `data` schema applies. */ +const ingestEvent = z.discriminatedUnion('type', [ + z.object({ type: z.literal('agent.heartbeat'), data: agentHeartbeatData }), + z.object({ type: z.literal('agent.deregister'), data: agentDeregisterData }), + z.object({ type: z.literal('observation'), data: observationData }), + z.object({ type: z.literal('prompt_use'), data: promptUseData }), + z.object({ type: z.literal('score'), data: scoreData }), +]); +export type IngestEvent = z.infer; + +/** The request body of `POST /api/v1/ingest`. */ +const ingestEnvelope = z.object({ + batch: z.array(ingestEvent).max(OBSERVE_MAX_BATCH_EVENTS), +}); +export type IngestEnvelope = z.infer; + +export type EnvelopeParseResult = { ok: true; envelope: IngestEnvelope } | { ok: false; issues: string[] }; + +/** + * Validates a decoded JSON body against the envelope. Returns issues rather + * than throwing: the daemon turns them into a `400` whose message names every + * offending `batch[i]` path (a reporter bug must be debuggable from the + * response alone). + */ +export function parseIngestEnvelope(value: unknown): EnvelopeParseResult { + const parsed = ingestEnvelope.safeParse(value); + if (parsed.success) return { ok: true, envelope: parsed.data }; + const issues = parsed.error.issues.map((issue) => `${issuePath(issue)}: ${issueMessage(issue)}`); + return { ok: false, issues }; +} diff --git a/src/observe/httpClient.ts b/src/observe/httpClient.ts new file mode 100644 index 00000000..3912b55d --- /dev/null +++ b/src/observe/httpClient.ts @@ -0,0 +1,43 @@ +/** + * One JSON call against the observe daemon, shared by the agent-side + * clients (hub, prompt): `fetch` with the bearer header when a token is + * set, a hard `timeoutMs` cap, and a non-2xx answer thrown as + * `LOUSHO_REMOTE_REQUEST_FAILED` carrying the daemon's `{ error }` message. + */ + +import { SDKError } from '../execution/errors'; + +/** Options of {@link observeRequest}. */ +export interface ObserveRequestOptions { + /** Bearer token for a `--require-token` daemon (`LOUSHO_OBSERVE_TOKEN`). */ + token?: string; + /** Injectable transport (tests); default the global `fetch`. */ + fetchFn?: typeof fetch; +} + +/** One `fetch` against the daemon as described above; resolves with the parsed JSON body. */ +export async function observeRequest( + baseUrl: string, + path: string, + init: RequestInit, + timeoutMs: number, + options: ObserveRequestOptions = {} +): Promise { + const res = await (options.fetchFn ?? fetch)(`${baseUrl.replace(/\/+$/, '')}${path}`, { + ...init, + headers: { + ...(options.token ? { authorization: `Bearer ${options.token}` } : {}), + ...(init.headers as Record | undefined), + }, + signal: AbortSignal.timeout(timeoutMs), + }); + const body = (await res.json().catch(() => undefined)) as ({ error?: string } & T) | undefined; + if (!res.ok) { + throw new SDKError( + `The observe daemon answered ${res.status} for ${path}: ${body?.error ?? res.statusText}.`, + 'LOUSHO_REMOTE_REQUEST_FAILED', + { appendHelp: false } + ); + } + return body as T; +} diff --git a/src/observe/hub.test.ts b/src/observe/hub.test.ts new file mode 100644 index 00000000..55345ff0 --- /dev/null +++ b/src/observe/hub.test.ts @@ -0,0 +1,172 @@ +/** + * The Context Hub end to end: publish/query over real HTTP, scope + * enforcement daemon side (private isolation, fleet visibility), upsert + * semantics per (namespace, agent_id, key), TTL expiry, and the SDK's + * `hub()`/`hubTools()` surface. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { startObserveDaemon, type ObserveDaemon } from './daemon'; +import { hub, hubTools, type HubEntry } from './hubClient'; + +const daemons: ObserveDaemon[] = []; +let cwd: string; + +beforeEach(() => { + cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-hub-')); +}); + +afterEach(async () => { + while (daemons.length > 0) await daemons.pop()?.close(); + fs.rmSync(cwd, { recursive: true, force: true }); +}); + +async function start(): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0 }); + daemons.push(daemon); + return daemon; +} + +const url = (daemon: ObserveDaemon, path: string): string => `http://127.0.0.1:${daemon.port}${path}`; +const base = (daemon: ObserveDaemon): string => url(daemon, ''); + +const publish = (daemon: ObserveDaemon, body: unknown): Promise => + fetch(url(daemon, '/api/v1/hub'), { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + +const query = async (daemon: ObserveDaemon, qs: string): Promise<{ entries: HubEntry[] }> => + (await (await fetch(url(daemon, `/api/v1/hub?${qs}`))).json()) as { entries: HubEntry[] }; + +describe('POST /api/v1/hub', () => { + it('publishes an entry and upserts the same (namespace, agent, key) in place', async () => { + const daemon = await start(); + const first = await publish(daemon, { agent_id: 'a', namespace: 'facts', key: 'sky', value: 'blue' }); + expect(first.status).toBe(201); + const again = await publish(daemon, { agent_id: 'a', namespace: 'facts', key: 'sky', value: 'gray', scope: 'fleet' }); + expect(again.status).toBe(200); + expect(((await again.json()) as { created: boolean }).created).toBe(false); + const { entries } = await query(daemon, 'agent_id=a'); + expect(entries).toHaveLength(1); + expect(entries[0].value).toBe('gray'); + expect(entries[0].scope).toBe('fleet'); + }); + + it('JSON-encodes non-string values and stores strings verbatim', async () => { + const daemon = await start(); + await publish(daemon, { agent_id: 'a', namespace: 'n', key: 'obj', value: { x: 1 } }); + const { entries } = await query(daemon, 'agent_id=a&key=obj'); + expect(entries[0].value).toBe('{"x":1}'); + }); + + it('rejects a bad body with 400', async () => { + const daemon = await start(); + expect((await publish(daemon, { agent_id: 'a', namespace: 'n' })).status).toBe(400); + expect((await publish(daemon, { agent_id: 'a', namespace: 'n', key: 'k', value: 'v', scope: 'world' })).status).toBe(400); + }); +}); + +describe('GET /api/v1/hub scope enforcement', () => { + it("keeps another agent's private entries invisible, shares fleet entries, and defaults to both kinds", async () => { + const daemon = await start(); + await publish(daemon, { agent_id: 'a', namespace: 'facts', key: 'mine', value: 'a-private' }); + await publish(daemon, { agent_id: 'a', namespace: 'facts', key: 'ours', value: 'a-fleet', scope: 'fleet' }); + await publish(daemon, { agent_id: 'b', namespace: 'facts', key: 'theirs', value: 'b-private' }); + await publish(daemon, { agent_id: 'b', namespace: 'facts', key: 'everyones', value: 'b-fleet', scope: 'fleet' }); + + const bPrivate = await query(daemon, 'agent_id=b&scope=private'); + expect(bPrivate.entries.map((e) => e.value)).toEqual(['b-private']); + + const bFleet = await query(daemon, 'agent_id=b&scope=fleet'); + expect(new Set(bFleet.entries.map((e) => e.value))).toEqual(new Set(['a-fleet', 'b-fleet'])); + + const bAll = await query(daemon, 'agent_id=b'); + expect(new Set(bAll.entries.map((e) => e.value))).toEqual(new Set(['b-private', 'a-fleet', 'b-fleet'])); + + const namespaced = await query(daemon, 'agent_id=b&namespace=other'); + expect(namespaced.entries).toHaveLength(0); + }); + + it('requires the caller agent_id and a known scope', async () => { + const daemon = await start(); + expect((await fetch(url(daemon, '/api/v1/hub'))).status).toBe(400); + expect((await fetch(url(daemon, '/api/v1/hub?agent_id=a&scope=world'))).status).toBe(400); + }); + + it('hides entries past their ttl', async () => { + const daemon = await start(); + await publish(daemon, { agent_id: 'a', namespace: 'n', key: 'gone', value: 'v', ttl: 30 }); + await publish(daemon, { agent_id: 'a', namespace: 'n', key: 'stays', value: 'v' }); + expect((await query(daemon, 'agent_id=a')).entries.map((e) => e.key).sort()).toEqual(['gone', 'stays']); + await new Promise((resolve) => setTimeout(resolve, 60)); + const { entries } = await query(daemon, 'agent_id=a'); + expect(entries.map((e) => e.key)).toEqual(['stays']); + }); +}); + +describe('hub() client', () => { + it('publishes and queries under the given agent identity', async () => { + const daemon = await start(); + const a = hub({ url: base(daemon), agentId: 'agent-a' })!; + const b = hub({ url: base(daemon), agentId: 'agent-b' })!; + await a.publish({ namespace: 'facts', key: 'sky', value: 'blue' }); + await a.publish({ namespace: 'facts', key: 'motto', value: 'share me', scope: 'fleet' }); + + expect((await a.query({ scope: 'private' })).map((e) => e.key)).toEqual(['sky']); + expect((await b.query({ scope: 'private' }))).toHaveLength(0); + expect((await b.query({ scope: 'fleet' })).map((e) => e.key)).toEqual(['motto']); + expect(a.agentId).toBe('agent-a'); + }); + + it('resolves the caller identity from .lousho/identity/ like the reporter', async () => { + const daemon = await start(); + const client = hub({ url: base(daemon), agentName: 'mira', project: 'testproj', cwd })!; + await client.publish({ namespace: 'n', key: 'k', value: 'v' }); + const { entries } = await query(daemon, `agent_id=${client.agentId}`); + expect(entries[0].agent_id).toBe(client.agentId); + expect(entries[0].agent_id).toMatch(/^[0-9a-f-]{36}$/); + }); + + it('is undefined when no daemon is configured', () => { + expect(hub({ env: {} as NodeJS.ProcessEnv })).toBeUndefined(); + }); + + it('follows LOUSHO_OBSERVE when no url is passed', async () => { + const daemon = await start(); + const client = hub({ env: { LOUSHO_OBSERVE: base(daemon) } as NodeJS.ProcessEnv, agentId: 'e-agent' })!; + await client.publish({ namespace: 'n', key: 'k', value: 'v' }); + expect((await query(daemon, 'agent_id=e-agent')).entries).toHaveLength(1); + }); +}); + +describe('hubTools()', () => { + it('returns [] when no hub is configured', () => { + expect(hubTools({ env: {} as NodeJS.ProcessEnv })).toEqual([]); + }); + + it('exposes hub_publish and hub_query as agent tools over the client', async () => { + const daemon = await start(); + const tools = hubTools({ url: base(daemon), agentId: 'tool-agent' }); + expect(tools.map((t) => t.name).sort()).toEqual(['hub_publish', 'hub_query']); + + const ctx = { + toolCallId: 't1', + messages: [], + getToken: async () => { + throw new Error('no oauth in tests'); + }, + requireAuth: () => { + throw new Error('no oauth in tests'); + }, + }; + const published = (await tools[0].execute({ namespace: 'facts', key: 'sky', value: 'blue', scope: 'private' }, ctx)) as { + ok: boolean; + entry: HubEntry; + }; + expect(published.ok).toBe(true); + expect(published.entry.agent_id).toBe('tool-agent'); + + const read = (await tools[1].execute({ scope: 'private' }, ctx)) as { entries: HubEntry[] }; + expect(read.entries.map((e) => e.key)).toEqual(['sky']); + }); +}); diff --git a/src/observe/hubClient.ts b/src/observe/hubClient.ts new file mode 100644 index 00000000..f51cfffb --- /dev/null +++ b/src/observe/hubClient.ts @@ -0,0 +1,235 @@ +/** + * The agent-side client of the Context Hub (spec 0001-prompts-context-hub): + * namespaced shared memory between agents on one daemon. + * + * `hub()` builds a `{publish, query}` client when `LOUSHO_OBSERVE` names a + * collector (or an explicit `url` is passed); `undefined` otherwise. Scope + * enforcement is the daemon's job - the client simply identifies itself with + * the same `agent_id` the reporter uses (`.lousho/identity/`), so an agent's + * `private` entries and its traces share one identity. + * + * `hubTools()` adapts the client into `hub_publish`/`hub_query` tools an + * agent can list in `tools` (`tools: [...hubTools()]`); it returns no tools + * when no hub is configured, matching the spec's "tools when LOUSHO_OBSERVE + * is set". + */ + +import { z } from 'zod'; +import { defineTool, type DefinedTool } from '../tools/defineTool'; +import { toolFailure } from '../tools/built-in/toolFailure'; +import { observeRequest } from './httpClient'; +import { resolveIdentity, resolveProject } from './identity'; +import { observeUrlFromEnv } from './reporter'; + +/** The scopes a hub entry can be published under (see {@link HubPublishInput.scope}). */ +export const HUB_SCOPES = ['private', 'fleet'] as const; +/** `'private'` is visible only to the publishing agent; `'fleet'` to every agent on the daemon. */ +export type HubScope = (typeof HUB_SCOPES)[number]; + +/** One `hub_entries` row as `GET /api/v1/hub` serves it (`expires_at` is derived: `created_at + ttl`). */ +export interface HubEntry { + /** The row's id, assigned by the daemon. */ + entry_id: string; + /** The group the entry lives in (e.g. 'facts', 'handoff'). */ + namespace: string; + /** The publishing agent's id. */ + agent_id: string | null; + /** The key inside the namespace. */ + key: string; + /** The stored value (JSON text for non-strings), or `null`. */ + value: string | null; + /** `'private'` or `'fleet'`. */ + scope: string; + /** Publish time (epoch ms). */ + created_at: number; + /** Lifetime in ms, or `null` when the entry does not expire. */ + ttl: number | null; + /** `created_at + ttl`, or `null` for non-expiring entries. */ + expires_at: number | null; +} + +/** Options of {@link hub} and {@link hubTools}. */ +export interface HubOptions { + /** Collector base URL; default `LOUSHO_OBSERVE`'s resolution. */ + url?: string; + /** + * The caller's `agent_id` (scope is enforced against it). Default: the + * reporter identity of `agentName`+project, the same id traces carry. + */ + agentId?: string; + /** Registry name the identity is resolved under (default `'agent'`; pass the agent's `name`). */ + agentName?: string; + /** Project label for identity resolution; default `LOUSHO_OBSERVE_PROJECT` else the cwd's package name. */ + project?: string; + /** Bearer token for a `--require-token` daemon (`LOUSHO_OBSERVE_TOKEN`). */ + token?: string; + /** Project dir for `.lousho/identity/`; default `process.cwd()`. */ + cwd?: string; + /** Request timeout (default 5000ms). */ + timeoutMs?: number; + /** Injectable transport (tests); default the global `fetch`. */ + fetchFn?: typeof fetch; + /** Injectable env (tests); default `process.env`. */ + env?: NodeJS.ProcessEnv; +} + +/** What {@link Hub.publish} stores. */ +export interface HubPublishInput { + /** The group the entry lives in (e.g. 'facts', 'handoff'). */ + namespace: string; + /** The key inside the namespace. */ + key: string; + /** A string is stored as-is; anything else is JSON-encoded. */ + value: unknown; + /** `'private'` (default) is visible only to this agent; `'fleet'` to every agent on the daemon. */ + scope?: HubScope; + /** Lifetime in milliseconds. */ + ttl?: number; +} + +/** The filters {@link Hub.query} accepts. */ +export interface HubQuery { + /** Only entries in this namespace; all namespaces when omitted. */ + namespace?: string; + /** `private` = only this agent's own private entries; `fleet` = all fleet entries; absent = both. */ + scope?: HubScope; + /** Only the entry with this key. */ + key?: string; + /** At most this many entries (default 100). */ + limit?: number; +} + +/** The Context Hub client {@link hub} returns. */ +export interface Hub { + /** The `agent_id` entries are published and scoped under (resolved on first call). */ + readonly agentId: string; + /** Store a value; resolves with the entry the daemon wrote. */ + publish(input: HubPublishInput): Promise; + /** Read entries matching `filter`; by default own private entries plus all fleet entries. */ + query(filter?: HubQuery): Promise; +} + +const DEFAULT_TIMEOUT_MS = 5000; + +/** + * A Context Hub client, or `undefined` when no daemon is configured (neither + * `options.url` nor a `LOUSHO_OBSERVE` env). Identity is resolved lazily - + * building a client must not write `.lousho/identity/` before the hub is used. + */ +export function hub(options: HubOptions = {}): Hub | undefined { + const env = options.env ?? (typeof process !== 'undefined' ? process.env : undefined); + const url = (options.url ?? observeUrlFromEnv(env))?.replace(/\/+$/, ''); + if (url === undefined) return undefined; + const token = options.token ?? env?.LOUSHO_OBSERVE_TOKEN; + const fetchFn = options.fetchFn ?? fetch; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const cwd = options.cwd ?? (typeof process !== 'undefined' ? process.cwd() : '.'); + + let id: string | undefined; + const agentId = (): string => { + id ??= + options.agentId ?? + resolveIdentity({ + name: options.agentName ?? 'agent', + project: resolveProject(options.project ?? env?.LOUSHO_OBSERVE_PROJECT, cwd), + cwd, + }).agentId; + return id; + }; + + const request = (path: string, init: RequestInit): Promise => + observeRequest(url, path, init, timeoutMs, { ...(token ? { token } : {}), fetchFn }); + + return { + get agentId() { + return agentId(); + }, + async publish(input) { + const body = await request<{ entry: HubEntry }>('/api/v1/hub', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ agent_id: agentId(), ...input }), + }); + return body.entry; + }, + async query(filter = {}) { + const params = new URLSearchParams({ agent_id: agentId() }); + if (filter.namespace !== undefined) params.set('namespace', filter.namespace); + if (filter.scope !== undefined) params.set('scope', filter.scope); + if (filter.key !== undefined) params.set('key', filter.key); + if (filter.limit !== undefined) params.set('limit', String(filter.limit)); + const body = await request<{ entries: HubEntry[] }>(`/api/v1/hub?${params}`, { method: 'GET' }); + return body.entries; + }, + }; +} + +/* ------------------------------------------------------------------ tools */ + +const publishInput = z.object({ + namespace: z.string().min(1).describe("The group the entry lives in, e.g. 'facts', 'handoff', 'preferences'."), + key: z.string().min(1).describe('The key inside the namespace.'), + value: z + .union([z.string(), z.number(), z.boolean(), z.record(z.string(), z.unknown()), z.array(z.unknown())]) + .describe('What to store; objects and arrays are JSON-encoded.'), + scope: z + .enum(['private', 'fleet']) + .optional() + .describe("'private' (default) is visible only to this agent; 'fleet' is visible to every agent on the daemon."), + ttl: z.number().int().positive().optional().describe('Optional lifetime in milliseconds; the entry expires afterwards.'), +}); + +const queryInput = z.object({ + namespace: z.string().optional().describe('Only entries in this namespace; all namespaces when omitted.'), + scope: z + .enum(['private', 'fleet']) + .optional() + .describe("'private' = only this agent's own private entries; 'fleet' = every agent's fleet entries; omitted = both."), + key: z.string().optional().describe('Only the entry with this key.'), + limit: z.number().int().positive().max(1000).optional().describe('At most this many entries (default 100).'), +}); + +/** + * `hub_publish` + `hub_query` tools over {@link hub}. Returns `[]` when no + * collector is configured (`LOUSHO_OBSERVE` unset and no `url` option) - the + * spec gives agents these tools only when the hub exists. A failed call + * becomes a tool error the model reads (`LOUSHO_TOOL_EXECUTION_FAILED`), it + * never fails the run. + */ +export function hubTools(options: HubOptions = {}): DefinedTool[] { + const client = hub(options); + if (client === undefined) return []; + return [ + defineTool({ + name: 'hub_publish', + description: + "Publish a value to the Context Hub, this machine's shared memory between agents. " + + "scope 'private' (default) is visible only to you; 'fleet' is visible to every agent on the daemon.", + input: publishInput, + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true }, + execute: async (args) => { + try { + const entry = await client.publish(args); + return { ok: true, entry }; + } catch (error) { + throw toolFailure(`hub_publish: ${error instanceof Error ? error.message : String(error)}`, error); + } + }, + }), + defineTool({ + name: 'hub_query', + description: + 'Read entries from the Context Hub. Filter by namespace, scope and key; ' + + "'private' returns only your own entries, 'fleet' returns what any agent shared.", + input: queryInput, + annotations: { readOnlyHint: true, destructiveHint: false }, + execute: async (args) => { + try { + return { entries: await client.query(args) }; + } catch (error) { + throw toolFailure(`hub_query: ${error instanceof Error ? error.message : String(error)}`, error); + } + }, + }), + ]; +} diff --git a/src/observe/identity.ts b/src/observe/identity.ts new file mode 100644 index 00000000..9f6c5e33 --- /dev/null +++ b/src/observe/identity.ts @@ -0,0 +1,187 @@ +/** + * Reporter identity: the stable `agent_id` and fallback `session_id` a + * reporting process carries across restarts. + * + * - `agent_id` is a generated UUID persisted in the PROJECT's `.lousho/` + * directory (not `~/.lousho` - reporters run in arbitrary project dirs): + * `.lousho/identity/@.json`. Same agent + project + dir → + * same identity, so a restarted process keeps its registry row. + * - `session_id` is `sha256(agent_id + cwd)` kept in the same file: it only + * applies to spans that carry no `gen_ai.conversation.id` (non-session + * runs), and it is what lets a restarted process rejoin its own session + * stream. + * + * This module is imported by `createAgent`/`AgentExecutor`, which also ship + * in the Worker bundle - so the fs/crypto/os/path builtins are loaded + * lazily through `process.getBuiltinModule` (the sqlite driver's pattern), + * and every node-only feature degrades to an in-memory identity instead of + * breaking non-Node runtimes. + */ + +/** The slices of the builtins used here, typed so the lazy loads stay honest. */ +interface FsLike { + existsSync(path: string): boolean; + readFileSync(path: string, encoding: 'utf8'): string; + mkdirSync(path: string, options: { recursive: boolean }): void; + writeFileSync(path: string, data: string, encoding: 'utf8'): void; +} +interface CryptoLike { + createHash(name: 'sha256'): { update(text: string): { digest(enc: 'hex'): string } }; +} +interface OsLike { + hostname(): string; +} +interface PathLike { + join(...parts: string[]): string; + resolve(...parts: string[]): string; +} + +// Named to match the worker leak check's exempted call pattern (see +// WORKER_SAFE_BUILTIN_PROBE in src/deploy/adapters/cloudflare.ts): the +// the builtin id literals below stay out of the flagged set only while they +// appear as this function's argument. Do not rename. +function getBuiltinModule(name: string): T | undefined { + // Reporters bundled for non-Node runtimes have no fs; identity falls back to memory. + if (typeof process === 'undefined') return undefined; + return (process.getBuiltinModule?.(name) as T | undefined) ?? undefined; +} + +const lazyFs = () => getBuiltinModule('node:fs') as FsLike | undefined; +const lazyCrypto = () => getBuiltinModule('node:crypto') as CryptoLike | undefined; +const lazyOs = () => getBuiltinModule('node:os') as OsLike | undefined; +const lazyPath = () => + (getBuiltinModule('node:path') as PathLike | undefined) ?? { + // POSIX-join fallback for runtimes without the path builtin (identity paths are relative anyway). + join: (...parts: string[]) => parts.filter(Boolean).join('/'), + resolve: (p: string) => p, + }; + +/** `@` made safe for a file name (`:` is illegal on Windows, `/` is a separator). */ +function safeFilePart(value: string): string { + return value.replace(/[^A-Za-z0-9._-]/g, '_') || 'default'; +} + +/** FNV-1a, for when the crypto builtin is unavailable; session ids are labels, not secrets. */ +function fallbackHash(text: string): string { + let h = 0x811c9dc5; + for (let i = 0; i < text.length; i++) { + h ^= text.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(16).padStart(8, '0').repeat(4); +} + +function sha256Hex(text: string): string { + const crypto = lazyCrypto(); + if (crypto) return crypto.createHash('sha256').update(text).digest('hex'); + return fallbackHash(text); +} + +export interface ReporterIdentity { + agentId: string; + /** The fallback `session_id` for observations that carry no `gen_ai.conversation.id`. */ + sessionId: string; + project: string; + /** The identity file, when the fs was available to persist it. */ + file?: string; +} + +const memoryIdentities = new Map(); + +/** + * The project label on every row this reporter emits: + * `LOUSHO_OBSERVE_PROJECT` (pass it in), else the cwd's package.json `name`, + * else 'default'. + */ +export function resolveProject(envProject: string | undefined, cwd: string): string { + if (envProject) return envProject; + const fs = lazyFs(); + const p = lazyPath(); + if (fs) { + try { + const pkgFile = p.join(cwd, 'package.json'); + if (fs.existsSync(pkgFile)) { + const pkg = JSON.parse(fs.readFileSync(pkgFile, 'utf8')) as { name?: unknown }; + if (typeof pkg.name === 'string' && pkg.name) return pkg.name; + } + } catch { + // an unreadable package.json is no reason to lose telemetry + } + } + return 'default'; +} + +function dirnameOf(file: string): string { + const slash = Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\')); + return slash > 0 ? file.slice(0, slash) : '.'; +} + +/** + * Load or mint this agent's identity under `/.lousho/identity/`. The + * file is written once; two reporters of one process share the in-memory map + * so they agree even before the first flush. + */ +export function resolveIdentity(options: { name: string; project: string; cwd: string }): ReporterIdentity { + const key = `${options.name}@${options.project}@${options.cwd}`; + const cached = memoryIdentities.get(key); + if (cached) return cached; + + const fs = lazyFs(); + const p = lazyPath(); + const file = fs + ? p.join(p.resolve(options.cwd), '.lousho', 'identity', `${safeFilePart(options.name)}@${safeFilePart(options.project)}.json`) + : undefined; + + if (fs && file) { + try { + if (fs.existsSync(file)) { + const stored = JSON.parse(fs.readFileSync(file, 'utf8')) as { agent_id?: unknown; session_id?: unknown }; + if (typeof stored.agent_id === 'string' && stored.agent_id) { + const identity: ReporterIdentity = { + agentId: stored.agent_id, + sessionId: + typeof stored.session_id === 'string' && stored.session_id + ? stored.session_id + : sha256Hex(`${stored.agent_id}:${options.cwd}`), + project: options.project, + file, + }; + memoryIdentities.set(key, identity); + return identity; + } + } + } catch { + // a corrupt identity file is replaced, never fatal + } + } + + const agentId = globalThis.crypto.randomUUID(); + const identity: ReporterIdentity = { + agentId, + sessionId: sha256Hex(`${agentId}:${options.cwd}`), + project: options.project, + ...(file ? { file } : {}), + }; + if (fs && file) { + try { + fs.mkdirSync(dirnameOf(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ agent_id: identity.agentId, session_id: identity.sessionId }, null, 2), 'utf8'); + } catch (error) { + // A lost identity means this process re-registers as a new agent on restart: warn, never throw into a run. + console.warn( + `[lousho] observe: could not persist agent identity at ${file} (${error instanceof Error ? error.message : String(error)}); this process re-registers as a new agent on restart.` + ); + } + } + memoryIdentities.set(key, identity); + return identity; +} + +/** `os.hostname()` when the runtime has it. */ +export function hostName(): string | undefined { + try { + return lazyOs()?.hostname(); + } catch { + return undefined; + } +} diff --git a/src/observe/promptClient.ts b/src/observe/promptClient.ts new file mode 100644 index 00000000..ed998899 --- /dev/null +++ b/src/observe/promptClient.ts @@ -0,0 +1,161 @@ +/** + * The agent-side client of the prompt registry (spec 0001-prompts-context-hub). + * + * `createAgent({ prompt: 'name@version' })` and `LOUSHO_PROMPT=name@version` + * opt a run into registry-owned instructions: the SDK resolves the ref + * against the daemon `LOUSHO_OBSERVE` points at (`GET + * /api/v1/prompts/:name/:version`, where `latest` is resolved daemon side), + * the fetched text replaces `instructions`, and `instructions` stays the + * embedded fallback for when the daemon cannot be reached. The publish/list/ + * history calls back `lousho prompt push|list|get`. + * + * `registryPrompt()` is the createAgent wiring: it starts the fetch when the + * agent is BUILT (the spec's "fetch at build"), memoizes it for the agent's + * lifetime, and hands the reporter a `prompt_use` ref once the fetch landed. + */ + +import { observeRequest } from './httpClient'; +import { observeUrlFromEnv, type PromptUseProvider } from './reporter'; + +/** A `name@version` reference; `version` is an integer or the daemon-side movable `latest`. */ +export interface PromptRef { + name: string; + version: number | 'latest'; +} + +/** `name@version`: a name without whitespace/`/`/`@`, then `@` and an integer or `latest`. */ +const REF_PATTERN = /^([^\s@/]{1,200})@(\d+|latest)$/; + +/** + * Parse `'name@3'`, `'name@latest'` into a {@link PromptRef}; `undefined` + * when the value is not a registry reference. A bare `'mira'` (no `@`) is + * deliberately NOT a ref: `prompt` is also the deprecated `instructions` + * alias, and only the unambiguous `name@version` shape opts into the registry. + */ +export function parsePromptRef(value: string | undefined): PromptRef | undefined { + if (value === undefined) return undefined; + const match = REF_PATTERN.exec(value.trim()); + if (!match) return undefined; + return { name: match[1], version: match[2] === 'latest' ? 'latest' : Number(match[2]) }; +} + +/** The daemon's `prompt` row as served (labels/variables decoded). */ +export interface ResolvedPrompt { + prompt_id: string; + name: string; + version: number; + text: string | null; + labels: unknown; + variables: unknown; + hash: string | null; + created_at: number; +} + +export interface PromptClientOptions { + /** Bearer token for a `--require-token` daemon (`LOUSHO_OBSERVE_TOKEN`). */ + token?: string; + /** Request timeout (default 2500ms); the prompt fetch must not stall a run for long. */ + timeoutMs?: number; + /** Injectable transport (tests); default the global `fetch`. */ + fetchFn?: typeof fetch; +} + +const DEFAULT_TIMEOUT_MS = 2500; + +/** Error body shape the daemon answers; the message is surfaced verbatim. */ +function request(baseUrl: string, path: string, init: RequestInit, options: PromptClientOptions): Promise { + return observeRequest(baseUrl, path, init, options.timeoutMs ?? DEFAULT_TIMEOUT_MS, options); +} + +/** `GET /api/v1/prompts/:name/:version`. Throws `LOUSHO_REMOTE_REQUEST_FAILED` on a non-200 (404 included). */ +export async function fetchPrompt(baseUrl: string, ref: PromptRef, options: PromptClientOptions = {}): Promise { + const body = await request<{ prompt: ResolvedPrompt }>( + baseUrl, + `/api/v1/prompts/${encodeURIComponent(ref.name)}/${ref.version}`, + { method: 'GET' }, + options + ); + return body.prompt; +} + +/** `POST /api/v1/prompts` - `lousho prompt push`. `created` is false on an identical republish (hash dedupe). */ +export async function publishPrompt( + baseUrl: string, + input: { name: string; text: string; labels?: string[] | Record; variables?: string[] }, + options: PromptClientOptions = {} +): Promise<{ prompt: ResolvedPrompt; created: boolean }> { + return request(baseUrl, '/api/v1/prompts', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(input) }, options); +} + +export interface PromptSummary extends ResolvedPrompt { + versions: number; + uses: number; +} + +/** `GET /api/v1/prompts` - one row per name (latest version + counts). */ +export async function listPrompts(baseUrl: string, filter: { name?: string; limit?: number } = {}, options: PromptClientOptions = {}): Promise { + const params = new URLSearchParams(); + if (filter.name !== undefined) params.set('name', filter.name); + if (filter.limit !== undefined) params.set('limit', String(filter.limit)); + const qs = params.toString(); + const body = await request<{ prompts: PromptSummary[] }>(baseUrl, `/api/v1/prompts${qs ? `?${qs}` : ''}`, { method: 'GET' }, options); + return body.prompts; +} + +/** + * The resolved-or-fallback prompt a `prompt` ref produces for one agent. The + * fetch starts at construction (agent build) and is memoized for the agent's + * lifetime; a failure warns once and every run then uses the embedded + * instructions. + */ +export interface RegistryPrompt { + /** The fetched text, or `fallback()` when the fetch failed. */ + text(fallback: () => Promise): Promise; + /** The `prompt_use` ref the reporter emits on the run's root span; `undefined` until the fetch landed. */ + use: PromptUseProvider; + /** The ref being resolved (for messages). */ + ref: PromptRef; +} + +/** + * Resolve `ref` (`'name@version'`) against the daemon `env`'s + * `LOUSHO_OBSERVE` points at. Returns `undefined` - the opt-in does not + * apply - when the env names no daemon; `LOUSHO_OBSERVE_TOKEN` is sent when set. + */ +export function registryPrompt( + ref: string, + options: { env?: NodeJS.ProcessEnv; fetchFn?: typeof fetch; timeoutMs?: number } = {} +): RegistryPrompt | undefined { + const parsed = parsePromptRef(ref); + const env = options.env ?? (typeof process !== 'undefined' ? process.env : undefined); + const url = observeUrlFromEnv(env); + if (parsed === undefined || url === undefined) return undefined; + const fetchOptions: PromptClientOptions = { + ...(env?.LOUSHO_OBSERVE_TOKEN ? { token: env.LOUSHO_OBSERVE_TOKEN } : {}), + ...(options.fetchFn ? { fetchFn: options.fetchFn } : {}), + ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), + }; + let warned = false; + let resolved: ResolvedPrompt | undefined; + const pending = (async () => { + try { + const prompt = await fetchPrompt(url, parsed, fetchOptions); + resolved = prompt; + return prompt; + } catch (error) { + if (!warned) { + warned = true; + console.warn( + `[lousho] prompt: could not resolve '${ref}' at ${url} (${error instanceof Error ? error.message : String(error)}); ` + + 'the embedded instructions are used instead.' + ); + } + return undefined; + } + })(); + return { + ref: parsed, + text: async (fallback) => (await pending)?.text ?? (await fallback()), + use: () => (resolved !== undefined ? { prompt_id: resolved.prompt_id, version: resolved.version } : undefined), + }; +} diff --git a/src/observe/prompts.test.ts b/src/observe/prompts.test.ts new file mode 100644 index 00000000..94b8822a --- /dev/null +++ b/src/observe/prompts.test.ts @@ -0,0 +1,222 @@ +/** + * The prompt registry end to end over real HTTP: publish/version/dedupe, + * `latest` resolved daemon side, history + list + `uses` counts, the + * agent-side client (`fetchPrompt`, `registryPrompt` fallback), and the + * reporter's `prompt_use` event emitted on the run's root span. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { startObserveDaemon, type ObserveDaemon } from './daemon'; +import { fetchPrompt, parsePromptRef, publishPrompt, registryPrompt } from './promptClient'; +import { observeExporter, type ObserveReporter } from './reporter'; +import type { IngestEvent } from './envelope'; +import type { Span } from '../execution/tracing'; + +const daemons: ObserveDaemon[] = []; +const reporters: ObserveReporter[] = []; +let cwd: string; + +beforeEach(() => { + cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-prompts-')); +}); + +afterEach(async () => { + await Promise.all(reporters.splice(0).map((r) => r.close().catch(() => undefined))); + while (daemons.length > 0) await daemons.pop()?.close(); + fs.rmSync(cwd, { recursive: true, force: true }); +}); + +async function start(opts: Parameters[0] = {}): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0, ...opts }); + daemons.push(daemon); + return daemon; +} + +const url = (daemon: ObserveDaemon, path: string): string => `http://127.0.0.1:${daemon.port}${path}`; +const base = (daemon: ObserveDaemon): string => url(daemon, ''); + +const post = (daemon: ObserveDaemon, path: string, body: unknown): Promise => + fetch(url(daemon, path), { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + +const get = (daemon: ObserveDaemon, path: string): Promise => fetch(url(daemon, path)); + +describe('POST /api/v1/prompts', () => { + it('publishes versions, dedupes an identical republish, and still versions a rollback', async () => { + const daemon = await start(); + const v1 = await post(daemon, '/api/v1/prompts', { name: 'mira', text: 'You are Mira.', labels: ['prod'], variables: ['who'] }); + expect(v1.status).toBe(201); + const p1 = (await v1.json()) as { prompt: { prompt_id: string; version: number; hash: string }; created: boolean }; + expect(p1.created).toBe(true); + expect(p1.prompt.version).toBe(1); + expect(typeof p1.prompt.hash).toBe('string'); + + const same = await post(daemon, '/api/v1/prompts', { name: 'mira', text: 'You are Mira.', labels: ['prod'], variables: ['who'] }); + expect(same.status).toBe(200); + expect((await same.json()) as { created: boolean }).toMatchObject({ created: false }); + + const v2 = await post(daemon, '/api/v1/prompts', { name: 'mira', text: 'You are Mira 2.' }); + expect((await v2.json()) as { prompt: { version: number } }).toMatchObject({ prompt: { version: 2 } }); + + // republishing the older text is a real new version (a rollback), not a dedupe hit + const rollback = await post(daemon, '/api/v1/prompts', { name: 'mira', text: 'You are Mira.', labels: ['prod'], variables: ['who'] }); + expect((await rollback.json()) as { prompt: { version: number }; created: boolean }).toMatchObject({ prompt: { version: 3 }, created: true }); + }); + + it('rejects a bad body (missing text, illegal name) with 400', async () => { + const daemon = await start(); + expect((await post(daemon, '/api/v1/prompts', { name: 'x' })).status).toBe(400); + expect((await post(daemon, '/api/v1/prompts', { name: 'has space', text: 't' })).status).toBe(400); + expect((await post(daemon, '/api/v1/prompts', { name: 'a/b', text: 't' })).status).toBe(400); + }); +}); + +describe('GET /api/v1/prompts reads', () => { + it('resolves name@version and name@latest, serves history and the one-row-per-name list', async () => { + const daemon = await start(); + await publishPrompt(base(daemon), { name: 'mira', text: 'v1 text' }); + await publishPrompt(base(daemon), { name: 'mira', text: 'v2 text' }); + await publishPrompt(base(daemon), { name: 'ghost', text: 'ghost v1' }); + + const latest = (await (await get(daemon, '/api/v1/prompts/mira/latest')).json()) as { prompt: { version: number; text: string } }; + expect(latest.prompt.version).toBe(2); + expect(latest.prompt.text).toBe('v2 text'); + + const pinned = (await (await get(daemon, '/api/v1/prompts/mira/1')).json()) as { prompt: { text: string } }; + expect(pinned.prompt.text).toBe('v1 text'); + + const history = (await (await get(daemon, '/api/v1/prompts/mira')).json()) as { versions: { version: number; text: string }[] }; + expect(history.versions.map((v) => [v.version, v.text])).toEqual([ + [2, 'v2 text'], + [1, 'v1 text'], + ]); + + const list = (await (await get(daemon, '/api/v1/prompts')).json()) as { prompts: { name: string; version: number; versions: number }[] }; + expect(list.prompts.map((p) => [p.name, p.version, p.versions])).toEqual([ + ['ghost', 1, 1], + ['mira', 2, 2], + ]); + }); + + it('404s a missing prompt or version and 400s a non-integer non-latest version', async () => { + const daemon = await start(); + await publishPrompt(base(daemon), { name: 'mira', text: 't' }); + expect((await get(daemon, '/api/v1/prompts/nope/latest')).status).toBe(404); + expect((await get(daemon, '/api/v1/prompts/mira/9')).status).toBe(404); + expect((await get(daemon, '/api/v1/prompts/nope')).status).toBe(404); + expect((await get(daemon, '/api/v1/prompts/mira/abc')).status).toBe(400); + }); + + it('counts prompt_uses rows as runs that used each version', async () => { + const daemon = await start(); + const { prompt } = await publishPrompt(base(daemon), { name: 'mira', text: 't' }); + await post(daemon, '/api/v1/ingest', { + batch: [ + { type: 'prompt_use', data: { obs_id: 'root-1', prompt_id: prompt.prompt_id, version: 1 } }, + { type: 'prompt_use', data: { obs_id: 'root-2', prompt_id: prompt.prompt_id, version: 1 } }, + ], + }); + const history = (await (await get(daemon, '/api/v1/prompts/mira')).json()) as { versions: { uses: number }[] }; + expect(history.versions[0].uses).toBe(2); + const list = (await (await get(daemon, '/api/v1/prompts')).json()) as { prompts: { uses: number }[] }; + expect(list.prompts[0].uses).toBe(2); + }); +}); + +describe('parsePromptRef + fetchPrompt', () => { + it('parses name@int and name@latest, rejects bare names and prose', () => { + expect(parsePromptRef('mira@3')).toEqual({ name: 'mira', version: 3 }); + expect(parsePromptRef('mira@latest')).toEqual({ name: 'mira', version: 'latest' }); + expect(parsePromptRef('mira')).toBeUndefined(); + expect(parsePromptRef('You are helpful.')).toBeUndefined(); + expect(parsePromptRef('a/b@1')).toBeUndefined(); + expect(parsePromptRef('@1')).toBeUndefined(); + expect(parsePromptRef(undefined)).toBeUndefined(); + }); + + it('fetchPrompt resolves a ref against the daemon', async () => { + const daemon = await start(); + await publishPrompt(base(daemon), { name: 'mira', text: 'hello' }); + const resolved = await fetchPrompt(base(daemon), { name: 'mira', version: 'latest' }); + expect(resolved.text).toBe('hello'); + await expect(fetchPrompt(base(daemon), { name: 'mira', version: 9 })).rejects.toMatchObject({ code: 'LOUSHO_REMOTE_REQUEST_FAILED' }); + }); +}); + +describe('registryPrompt', () => { + it('serves the fetched text and exposes the prompt_use ref', async () => { + const daemon = await start(); + const { prompt } = await publishPrompt(base(daemon), { name: 'mira', text: 'registry text' }); + const env = { LOUSHO_OBSERVE: base(daemon) } as NodeJS.ProcessEnv; + const source = registryPrompt('mira@latest', { env }); + expect(source).toBeDefined(); + await expect(source!.text(async () => 'embedded')).resolves.toBe('registry text'); + expect(source!.use()).toEqual({ prompt_id: prompt.prompt_id, version: 1 }); + }); + + it('falls back to the embedded instructions (warn once) when the daemon is unreachable', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const env = { LOUSHO_OBSERVE: 'http://127.0.0.1:1' } as NodeJS.ProcessEnv; + const source = registryPrompt('mira@1', { env, timeoutMs: 300 })!; + await expect(source.text(async () => 'embedded')).resolves.toBe('embedded'); + await expect(source.text(async () => 'embedded')).resolves.toBe('embedded'); + expect(source.use()).toBeUndefined(); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + }); + + it('is undefined when LOUSHO_OBSERVE names no daemon or the value is not a ref', () => { + expect(registryPrompt('mira@1', { env: {} as NodeJS.ProcessEnv })).toBeUndefined(); + expect(registryPrompt('not a ref', { env: { LOUSHO_OBSERVE: 'http://x' } as NodeJS.ProcessEnv })).toBeUndefined(); + }); +}); + +describe('reporter prompt_use', () => { + function reporter(over: Parameters[0] = {}): { r: ObserveReporter; posted: IngestEvent[] } { + const posted: IngestEvent[] = []; + const fetchFn = vi.fn(async (_input: unknown, init?: RequestInit): Promise => { + posted.push(...(JSON.parse(String(init?.body)) as { batch: IngestEvent[] }).batch); + return new Response('{}', { status: 202 }); + }); + const r = observeExporter({ + cwd, + url: 'http://127.0.0.1:1', + flushIntervalMs: 0, + heartbeatIntervalMs: 0, + registerExitHooks: false, + fetchFn: fetchFn as unknown as typeof fetch, + ...over, + }); + reporters.push(r); + return { r, posted }; + } + + const span = (over: Partial): Span => ({ id: 's', name: 'invoke_agent a', attributes: {}, startTime: 1, ...over }); + + it('emits prompt_use with obs_id = the root span id when the provider resolves', async () => { + const { r, posted } = reporter({ promptUse: () => ({ prompt_id: 'p-1', version: 2 }) }); + r.onSpanStart(span({ id: 'root' })); + r.onSpanStart(span({ id: 'child', parentId: 'root' })); + r.onSpanEnd(span({ id: 'child', parentId: 'root', endTime: 5 })); + r.onSpanEnd(span({ id: 'root', endTime: 6 })); + await r.close(); + const uses = posted.filter((e): e is Extract => e.type === 'prompt_use'); + expect(uses).toHaveLength(1); + expect(uses[0].data).toEqual({ obs_id: 'root', prompt_id: 'p-1', version: 2 }); + }); + + it('emits nothing while the provider returns undefined (fetch pending or failed)', async () => { + const { r, posted } = reporter({ promptUse: () => undefined }); + r.onSpanStart(span({ id: 'root' })); + await r.close(); + expect(posted.filter((e) => e.type === 'prompt_use')).toHaveLength(0); + }); + + it('does not re-emit for a child run parented under a span', async () => { + const { r, posted } = reporter({ promptUse: () => ({ prompt_id: 'p-1' }) }); + r.onSpanStart(span({ id: 'delegated', parentId: 'unseen-parent' })); + await r.close(); + expect(posted.filter((e) => e.type === 'prompt_use')).toHaveLength(0); + }); +}); diff --git a/src/observe/reporter.test.ts b/src/observe/reporter.test.ts new file mode 100644 index 00000000..dfc52286 --- /dev/null +++ b/src/observe/reporter.test.ts @@ -0,0 +1,341 @@ +/** + * The reporter's contract: finished spans become `observation` events + * (span.id is obs_id, parentage gives the trace), flushes are size/interval/ + * exit-triggered and serialized, an unreachable daemon or a full buffer + * drops-and-counts without ever throwing into the run, and the env + * (`LOUSHO_OBSERVE`) wiring composes with an explicit exporter. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { GenAiAttr, GenAiOperation, LegacyAttr, SdkAttr } from '../execution/semconv'; +import type { Span, TraceExporter } from '../execution/tracing'; +import type { IngestEvent } from './envelope'; +import { fanOutExporters, observeExporter, observeExporterFromEnv, withObserveExporter, type ObserveReporter } from './reporter'; + +interface PostedBatch { + url: string; + authorization?: string; + events: IngestEvent[]; +} + +/** A fetch mock that records every batch and answers 202 (or `fail`, to emulate a dead daemon). */ +function mockFetch(fail = false): { fetchFn: typeof fetch; posted: PostedBatch[] } { + const posted: PostedBatch[] = []; + const fetchFn = vi.fn(async (input: string | URL | Request, init?: RequestInit): Promise => { + const url = String(input); + const body = JSON.parse(String(init?.body)) as { batch: IngestEvent[] }; + const headers = (init?.headers ?? {}) as Record; + posted.push({ url, authorization: headers.authorization, events: body.batch }); + return new Response('{}', { status: fail ? 500 : 202 }); + }); + return { fetchFn: fetchFn as unknown as typeof fetch, posted }; +} + +const reporters: ObserveReporter[] = []; +let cwd: string; + +function makeSpan(over: Partial = {}): Span { + return { id: `s${Math.random().toString(36).slice(2)}`, name: 'invoke_agent a', attributes: {}, startTime: 100, endTime: 150, ...over }; +} + +function reporter(over: Parameters[0] = {}, fail = false): { r: ObserveReporter; posted: PostedBatch[] } { + const { fetchFn, posted } = mockFetch(fail); + const r = observeExporter({ + cwd, + url: 'http://127.0.0.1:1', + project: 'testproj', + agentName: 'agent-a', + flushIntervalMs: 0, + heartbeatIntervalMs: 0, + registerExitHooks: false, + fetchFn, + ...over, + }); + reporters.push(r); + return { r, posted }; +} + +const allEvents = (posted: PostedBatch[]): IngestEvent[] => posted.flatMap((p) => p.events); +const observations = (posted: PostedBatch[]) => + allEvents(posted).filter((e): e is Extract => e.type === 'observation'); + +beforeEach(() => { + cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-reporter-')); +}); + +afterEach(async () => { + await Promise.all(reporters.splice(0).map((r) => r.close().catch(() => undefined))); + fs.rmSync(cwd, { recursive: true, force: true }); +}); + +describe('observeExporter', () => { + it('emits one observation per finished span, trace-grouped by parentage', async () => { + const { r, posted } = reporter(); + const root = makeSpan({ id: 'root', attributes: { [GenAiAttr.OPERATION_NAME]: GenAiOperation.INVOKE_AGENT } }); + const child = makeSpan({ + id: 'child', + parentId: 'root', + name: 'execute_tool x', + startTime: 110, + endTime: 130, + status: { code: 'ok' }, + attributes: { [GenAiAttr.OPERATION_NAME]: GenAiOperation.EXECUTE_TOOL, [GenAiAttr.TOOL_CALL_ARGUMENTS]: '{"a":1}', [GenAiAttr.TOOL_CALL_RESULT]: 'done' }, + }); + r.onSpanStart(root); + r.onSpanStart(child); + r.onSpanEnd(child); + r.onSpanEnd(root); + await r.close(); + + const obs = observations(posted); + expect(obs).toHaveLength(2); + const byRoot = obs.find((o) => o.data.obs_id === 'root')?.data; + const byChild = obs.find((o) => o.data.obs_id === 'child')?.data; + expect(byRoot?.trace_id).toBe('root'); + expect(byChild?.trace_id).toBe('root'); + expect(byChild?.parent_obs_id).toBe('root'); + expect(byChild?.kind).toBe('tool'); + expect(byChild?.input).toBe('{"a":1}'); + expect(byChild?.output).toBe('done'); + expect(byChild?.agent_id).toBe(r.agentId); + expect(byChild?.session_id).toBe(r.sessionId); + // the final batch ends with the deregister event + expect(allEvents(posted).at(-1)?.type).toBe('agent.deregister'); + }); + + it('carries usage/cost/flags verbatim from gen_ai + lousho attrs', async () => { + const { r, posted } = reporter(); + const span = makeSpan({ + id: 'chat1', + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperation.CHAT, + [GenAiAttr.RESPONSE_MODEL]: 'm-1', + [GenAiAttr.USAGE_INPUT_TOKENS]: 100, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: 40, + 'gen_ai.usage.cached_input_tokens': 30, + [SdkAttr.USAGE_REASONING_TOKENS]: 12, + [SdkAttr.COST_USD]: 0.007, + 'lousho.cost_partial': true, + [SdkAttr.USAGE_ESTIMATED]: true, + [GenAiAttr.CONVERSATION_ID]: 'real-session', + }, + }); + r.onSpanStart(span); + r.onSpanEnd(span); + await r.close(); + + const d = observations(posted)[0].data; + expect(d.kind).toBe('generation'); + expect(d.model).toBe('m-1'); + expect(d.usage_input_tokens).toBe(100); + expect(d.usage_output_tokens).toBe(40); + expect(d.usage_cached_tokens).toBe(30); + expect(d.usage_reasoning_tokens).toBe(12); + expect(d.cost_usd).toBe(0.007); + expect(d.cost_partial).toBe(true); + expect(d.usage_estimated).toBe(true); + // a real conversation id beats the fallback session + expect(d.session_id).toBe('real-session'); + }); + + it('falls back to legacy attribute names', async () => { + const { r, posted } = reporter(); + const span = makeSpan({ id: 'legacy', attributes: { [LegacyAttr.MODEL]: 'old-m', [LegacyAttr.PROMPT_TOKENS]: 9, input: 'hi', result: 'yo' } }); + r.onSpanStart(span); + r.onSpanEnd(span); + await r.close(); + const d = observations(posted)[0].data; + expect(d.model).toBe('old-m'); + expect(d.usage_input_tokens).toBe(9); + expect(d.input).toBe('hi'); + expect(d.output).toBe('yo'); + }); + + it('flushes when the buffer reaches flushSize', async () => { + const { r, posted } = reporter({ flushSize: 3 }); + for (let i = 0; i < 3; i++) { + const span = makeSpan({ id: `s${i}` }); + r.onSpanStart(span); + r.onSpanEnd(span); + } + // the size flush is chained microtask work - one turn is enough + await vi.waitFor(() => expect(observations(posted)).toHaveLength(3)); + await r.close(); + }); + + it('drops and counts when the daemon is unreachable, warning once, never throwing', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + try { + const { r, posted } = reporter({}, true); + const span = makeSpan(); + r.onSpanStart(span); + r.onSpanEnd(span); + await expect(r.close()).resolves.toBeUndefined(); + expect(r.dropped).toBeGreaterThanOrEqual(1); + expect(posted.length).toBeGreaterThanOrEqual(1); // the POST was attempted + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } + }); + + it('drops events past maxBuffered', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + try { + const { r } = reporter({ maxBuffered: 2, flushSize: 100 }, true); + for (let i = 0; i < 5; i++) { + const span = makeSpan(); + r.onSpanStart(span); + r.onSpanEnd(span); + } + expect(r.dropped).toBe(3); + await r.close(); + } finally { + warn.mockRestore(); + } + }); + + it('emits a heartbeat immediately when heartbeats are on', async () => { + const { r, posted } = reporter({ heartbeatIntervalMs: 60_000 }); + await r.close(); + const hb = allEvents(posted).find((e) => e.type === 'agent.heartbeat'); + expect(hb).toBeDefined(); + expect(hb?.type === 'agent.heartbeat' && hb.data.project).toBe('testproj'); + }); + + it('sends the bearer token when configured', async () => { + const { r, posted } = reporter({ token: 'lso_abc' }); + const span = makeSpan(); + r.onSpanStart(span); + r.onSpanEnd(span); + await r.close(); + expect(posted.every((p) => p.authorization === 'Bearer lso_abc')).toBe(true); + }); + + it('persists agent_id + session_id across reporters in one project dir', async () => { + const first = reporter(); + const second = reporter(); + expect(first.r.agentId).toBe(second.r.agentId); + expect(first.r.sessionId).toBe(second.r.sessionId); + await first.r.close(); + await second.r.close(); + // and across "restarts" via the identity file + const file = path.join(cwd, '.lousho', 'identity', 'agent-a@testproj.json'); + expect(fs.existsSync(file)).toBe(true); + const stored = JSON.parse(fs.readFileSync(file, 'utf8')) as { agent_id: string }; + expect(stored.agent_id).toBe(first.r.agentId); + }); + + it('close() is idempotent and posts the deregister exactly once', async () => { + const { r, posted } = reporter(); + await r.close(); + await r.close(); + expect(allEvents(posted).filter((e) => e.type === 'agent.deregister')).toHaveLength(1); + }); + + it('counts a span ending after close() as dropped', async () => { + const { r } = reporter(); + await r.close(); + const span = makeSpan(); + r.onSpanStart(span); + r.onSpanEnd(span); + expect(r.dropped).toBe(1); + }); +}); + +describe('env wiring', () => { + it('returns undefined when LOUSHO_OBSERVE is unset or off', () => { + expect(observeExporterFromEnv({ env: {} })).toBeUndefined(); + expect(observeExporterFromEnv({ env: { LOUSHO_OBSERVE: '0' } })).toBeUndefined(); + expect(observeExporterFromEnv({ env: { LOUSHO_OBSERVE: 'false' } })).toBeUndefined(); + }); + + it('maps =1/true to the loopback default and passes a URL through', async () => { + const { fetchFn, posted } = mockFetch(); + vi.stubGlobal('fetch', fetchFn); + try { + const a = observeExporterFromEnv({ agentName: 'env-a', env: { LOUSHO_OBSERVE: '1', LOUSHO_OBSERVE_PORT: '8877' } }); + expect(a).toBeDefined(); + await a!.close(); + expect(posted.at(-1)?.url).toBe('http://127.0.0.1:8877/api/v1/ingest'); + const b = observeExporterFromEnv({ agentName: 'env-b', env: { LOUSHO_OBSERVE: 'http://example.test:9999' } }); + await b!.close(); + expect(posted.at(-1)?.url).toBe('http://example.test:9999/api/v1/ingest'); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('memoizes per (name, project, url)', () => { + const { fetchFn } = mockFetch(); + vi.stubGlobal('fetch', fetchFn); + try { + const env = { LOUSHO_OBSERVE: '1' }; + expect(observeExporterFromEnv({ agentName: 'memo-a', env })).toBe(observeExporterFromEnv({ agentName: 'memo-a', env })); + expect(observeExporterFromEnv({ agentName: 'memo-a', env })).not.toBe(observeExporterFromEnv({ agentName: 'memo-b', env })); + } finally { + vi.unstubAllGlobals(); + } + }); +}); + +describe('fanOutExporters / withObserveExporter', () => { + const counting = (): { exporter: TraceExporter; spans: string[] } => { + const spans: string[] = []; + return { spans, exporter: { onSpanStart: (s) => void spans.push(`start:${s.id}`), onSpanEnd: (s) => void spans.push(`end:${s.id}`) } }; + }; + + it('fans every span out to each exporter and dedupes', () => { + const a = counting(); + const b = counting(); + const fan = fanOutExporters([a.exporter, b.exporter, a.exporter, undefined]); + const span = makeSpan({ id: 'x' }); + fan?.onSpanStart(span); + fan?.onSpanEnd(span); + expect(a.spans).toEqual(['start:x', 'end:x']); + expect(b.spans).toEqual(['start:x', 'end:x']); + // re-composing a fan with one of its parts still delivers once each + const fan2 = fanOutExporters([fan, a.exporter]); + fan2?.onSpanStart(makeSpan({ id: 'y' })); + expect(a.spans).toEqual(['start:x', 'end:x', 'start:y']); + expect(b.spans.filter((s) => s === 'start:y')).toHaveLength(1); + }); + + it('keeps one exporter seeing spans when another throws', () => { + const good = counting(); + const bad: TraceExporter = { + onSpanStart: () => { + throw new Error('boom'); + }, + onSpanEnd: () => { + throw new Error('boom'); + }, + }; + const fan = fanOutExporters([bad, good.exporter]); + const span = makeSpan(); + expect(() => fan?.onSpanStart(span)).toThrow('boom'); + expect(() => fan?.onSpanEnd(span)).toThrow('boom'); + expect(good.spans).toEqual([`start:${span.id}`, `end:${span.id}`]); + }); + + it('env-composes an explicit exporter once (no double env reporter)', () => { + const env = { LOUSHO_OBSERVE: '1', LOUSHO_OBSERVE_PORT: '3' }; + const explicit = counting(); + vi.stubEnv('LOUSHO_OBSERVE', env.LOUSHO_OBSERVE); + vi.stubEnv('LOUSHO_OBSERVE_PORT', env.LOUSHO_OBSERVE_PORT); + try { + const composed = withObserveExporter(explicit.exporter, 'combo-a'); + expect(composed).toBeDefined(); + // composing an already env-composed exporter returns it unchanged + expect(withObserveExporter(composed, 'combo-a')).toBe(composed); + const span = makeSpan({ id: 'z' }); + composed?.onSpanStart(span); + composed?.onSpanEnd(span); + expect(explicit.spans).toEqual(['start:z', 'end:z']); + } finally { + vi.unstubAllEnvs(); + } + }); +}); diff --git a/src/observe/reporter.ts b/src/observe/reporter.ts new file mode 100644 index 00000000..4f665447 --- /dev/null +++ b/src/observe/reporter.ts @@ -0,0 +1,527 @@ +/** + * `observeExporter()` - the SDK's reporter: a `TraceExporter` that drains + * finished spans into batched `POST /api/v1/ingest` calls to the collector + * daemon (`lousho observe start`). Wire it explicitly, or let the + * `LOUSHO_OBSERVE` env do it with zero code changes: + * + * - `LOUSHO_OBSERVE=1`/`true` → `http://127.0.0.1:$LOUSHO_OBSERVE_PORT(7777)` + * - `LOUSHO_OBSERVE=` → that collector + * - unset/`0`/`false` → nothing is emitted + * - `LOUSHO_OBSERVE_PROJECT` labels rows; `LOUSHO_OBSERVE_TOKEN` sends a bearer. + * + * Rules it lives by (the tracing-insights spec): + * - One event per FINISHED span (`start`+`end`); `obs_id` is the span's own + * id (the `TraceLine.id`), so a resent batch upserts by key. + * - Flush on size (50), on interval (5s), and on process exit (~2s cap). + * Crash loss is accepted and bounded: nothing persists mid run. + * - The daemon being down, or a full buffer, drops events into a `dropped` + * counter and warns ONCE - the reporter never blocks or throws into a run. + * Timers are unref'd so observation never keeps a process alive. + * - Its own HTTP calls are plain `fetch`, outside any span: instrumenting + * them would recurse. + * + * Identity: `agent_id`/`session_id` come from `.lousho/identity/` in the + * project dir (see identity.ts); `session_id` is the span's + * `gen_ai.conversation.id` when the run has a real session. + */ + +import { GenAiAttr, GenAiOperation, LegacyAttr, SdkAttr } from '../execution/semconv'; +import { SDKError } from '../execution/errors'; +import type { Span, TraceExporter } from '../execution/tracing'; +import { VERSION } from '../version'; +import { OBSERVE_DEFAULT_PORT, type IngestEvent, type ObservationData, type ObservationKind, type PromptUseData } from './envelope'; +import { hostName, resolveIdentity, resolveProject } from './identity'; + +/** The registry prompt a run used (`prompt_use` minus its `obs_id`, which the reporter fills with the root span's id). */ +export type PromptUseRef = Pick; + +/** Resolves the run's registry prompt once fetched; `undefined` means no `prompt_use` is emitted (fetch pending or failed). */ +export type PromptUseProvider = () => PromptUseRef | undefined; + +/** Options of {@link observeExporter}; every field defaults so `observeExporter()` alone targets the loopback daemon. */ +export interface ObserveExporterOptions { + /** Collector base URL (default `http://127.0.0.1:7777`). */ + url?: string; + /** Project label on every row; default `LOUSHO_OBSERVE_PROJECT` else the cwd's package.json name. */ + project?: string; + /** Registry name; default `'agent'`. The env wiring passes the createAgent config's `name`. */ + agentName?: string; + /** Bearer token for a daemon that demands one (`LOUSHO_OBSERVE_TOKEN`). */ + token?: string; + /** Explicit fallback `session_id`; default `sha256(agent_id+cwd)` from the identity file. */ + sessionId?: string; + /** Flush when this many events are buffered (default 50). */ + flushSize?: number; + /** Flush interval (default 5000ms). */ + flushIntervalMs?: number; + /** Heartbeat cadence (default 60000ms); `0` disables heartbeats. */ + heartbeatIntervalMs?: number; + /** Buffer cap; events past it drop into `dropped` (default 5000). */ + maxBuffered?: number; + /** Per-request timeout (default 5000ms); the exit flush always caps at ~2000ms. */ + requestTimeoutMs?: number; + /** Project dir for `.lousho/identity/`; default `process.cwd()`. */ + cwd?: string; + /** Version reported on heartbeats; default the SDK's. */ + version?: string; + /** Injectable transport (tests); default the global `fetch`. */ + fetchFn?: typeof fetch; + /** Register process exit hooks (deregister + final flush); default true. */ + registerExitHooks?: boolean; + /** Injectable clock (tests); default `Date.now`. */ + now?: () => number; + /** + * The registry prompt this agent's runs resolved (the `prompt` opt-in). + * When a span with no parent starts - a run's root `invoke_agent` - and + * this provider returns a ref, a `prompt_use` event is queued with + * `obs_id` = that root span's id. + */ + promptUse?: PromptUseProvider; +} + +export interface ObserveReporter extends TraceExporter { + /** Events dropped because the daemon was unreachable or the buffer was full. */ + readonly dropped: number; + /** This reporter's persisted `agent_id`. */ + readonly agentId: string; + /** The fallback `session_id` (non-session spans). */ + readonly sessionId: string; + /** + * The registry prompt ref provider (see {@link ObserveExporterOptions.promptUse}). + * Mutable because `LOUSHO_OBSERVE` reporters are memoized per agent + * name: a later {@link observeExporterFromEnv} call with a prompt sets it + * on the shared reporter. + */ + promptUse?: PromptUseProvider; + /** Deregister, flush once, and release timers/hooks. Idempotent. */ + close(): Promise; +} + +const DEFAULT_FLUSH_SIZE = 50; +const DEFAULT_FLUSH_INTERVAL_MS = 5000; +const DEFAULT_HEARTBEAT_MS = 60_000; +const DEFAULT_MAX_BUFFERED = 5000; +const DEFAULT_REQUEST_TIMEOUT_MS = 5000; +const EXIT_FLUSH_TIMEOUT_MS = 2000; + +const num = (value: unknown): number | undefined => + typeof value === 'number' && Number.isFinite(value) ? value : undefined; + +const str = (value: unknown): string | undefined => (typeof value === 'string' && value ? value : undefined); + +/** JSON for a non-string attribute value (args/result objects); strings pass through. */ +function textOf(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value === 'string') return value; + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} + +/** The first defined attribute among `keys`, as a string. */ +function pickAttr(attrs: Record, keys: readonly string[]): string | undefined { + for (const key of keys) { + const text = textOf(attrs[key]); + if (text !== undefined) return text; + } + return undefined; +} + +const KIND_OF: Record = { + [GenAiOperation.CHAT]: 'generation', + [GenAiOperation.EXECUTE_TOOL]: 'tool', + [GenAiOperation.INVOKE_AGENT]: 'span', + [GenAiOperation.INVOKE_WORKFLOW]: 'span', +}; + +/** Which attributes carry a span's payload, per operation. */ +const INPUT_KEYS: Record = { + [GenAiOperation.CHAT]: [GenAiAttr.INPUT_MESSAGES, LegacyAttr.PROMPT], + [GenAiOperation.EXECUTE_TOOL]: [GenAiAttr.TOOL_CALL_ARGUMENTS, LegacyAttr.ARGS], + [GenAiOperation.INVOKE_AGENT]: [GenAiAttr.INPUT_MESSAGES, LegacyAttr.INPUT], +}; +const OUTPUT_KEYS: Record = { + [GenAiOperation.CHAT]: [GenAiAttr.OUTPUT_MESSAGES], + [GenAiOperation.EXECUTE_TOOL]: [GenAiAttr.TOOL_CALL_RESULT, LegacyAttr.RESULT], + [GenAiOperation.INVOKE_AGENT]: [GenAiAttr.OUTPUT_MESSAGES], +}; +const GENERIC_INPUT_KEYS = [GenAiAttr.INPUT_MESSAGES, LegacyAttr.INPUT] as const; +const GENERIC_OUTPUT_KEYS = [GenAiAttr.OUTPUT_MESSAGES, LegacyAttr.RESULT] as const; + +/** One finished span → one `observation` event (StepUsage fields carried verbatim from the gen_ai/lousho attrs). */ +function observationOf(span: Span, traceId: string, agentId: string, fallbackSessionId: string, now: () => number): ObservationData { + const attrs = span.attributes; + const op = str(attrs[GenAiAttr.OPERATION_NAME]) ?? ''; + const status = span.status?.code; + return { + obs_id: span.id, + trace_id: traceId, + session_id: str(attrs[GenAiAttr.CONVERSATION_ID]) ?? fallbackSessionId, + agent_id: agentId, + ...(span.parentId !== undefined ? { parent_obs_id: span.parentId } : {}), + kind: KIND_OF[op] ?? 'span', + name: span.name, + start: span.startTime, + end: span.endTime ?? now(), + ...(status === 'ok' || status === 'error' ? { status } : {}), + ...(pickAttr(attrs, INPUT_KEYS[op] ?? GENERIC_INPUT_KEYS) !== undefined + ? { input: pickAttr(attrs, INPUT_KEYS[op] ?? GENERIC_INPUT_KEYS) } + : {}), + ...(pickAttr(attrs, OUTPUT_KEYS[op] ?? GENERIC_OUTPUT_KEYS) !== undefined + ? { output: pickAttr(attrs, OUTPUT_KEYS[op] ?? GENERIC_OUTPUT_KEYS) } + : {}), + ...(str(attrs[GenAiAttr.RESPONSE_MODEL] ?? attrs[GenAiAttr.REQUEST_MODEL] ?? attrs[LegacyAttr.MODEL]) !== undefined + ? { model: str(attrs[GenAiAttr.RESPONSE_MODEL] ?? attrs[GenAiAttr.REQUEST_MODEL] ?? attrs[LegacyAttr.MODEL]) } + : {}), + ...(num(attrs[GenAiAttr.USAGE_INPUT_TOKENS] ?? attrs[LegacyAttr.PROMPT_TOKENS]) !== undefined + ? { usage_input_tokens: num(attrs[GenAiAttr.USAGE_INPUT_TOKENS] ?? attrs[LegacyAttr.PROMPT_TOKENS]) } + : {}), + ...(num(attrs[GenAiAttr.USAGE_OUTPUT_TOKENS] ?? attrs[LegacyAttr.COMPLETION_TOKENS]) !== undefined + ? { usage_output_tokens: num(attrs[GenAiAttr.USAGE_OUTPUT_TOKENS] ?? attrs[LegacyAttr.COMPLETION_TOKENS]) } + : {}), + ...(num(attrs['gen_ai.usage.cached_input_tokens']) !== undefined + ? { usage_cached_tokens: num(attrs['gen_ai.usage.cached_input_tokens']) } + : {}), + ...(num(attrs[SdkAttr.USAGE_REASONING_TOKENS]) !== undefined + ? { usage_reasoning_tokens: num(attrs[SdkAttr.USAGE_REASONING_TOKENS]) } + : {}), + ...(num(attrs[SdkAttr.COST_USD]) !== undefined ? { cost_usd: num(attrs[SdkAttr.COST_USD]) } : {}), + ...(attrs['lousho.cost_partial'] === true ? { cost_partial: true } : {}), + ...(attrs[SdkAttr.USAGE_ESTIMATED] === true ? { usage_estimated: true } : {}), + attrs, + }; +} + +/** + * A `TraceExporter` that batches finished spans to a collector daemon. The + * returned object is also an {@link ObserveReporter}: `close()` sends + * `agent.deregister` and the final flush. + */ +export function observeExporter(options: ObserveExporterOptions = {}): ObserveReporter { + const cwd = options.cwd ?? (typeof process !== 'undefined' ? process.cwd() : '.'); + const project = resolveProject(options.project, cwd); + const agentName = options.agentName ?? 'agent'; + const identity = resolveIdentity({ name: agentName, project, cwd }); + const fallbackSessionId = options.sessionId ?? identity.sessionId; + const url = (options.url ?? `http://127.0.0.1:${OBSERVE_DEFAULT_PORT}`).replace(/\/+$/, ''); + const token = options.token; + const version = options.version ?? VERSION; + const fetchFn = options.fetchFn ?? fetch; + const now = options.now ?? Date.now; + const flushSize = options.flushSize ?? DEFAULT_FLUSH_SIZE; + const flushIntervalMs = options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS; + const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_MS; + const maxBuffered = options.maxBuffered ?? DEFAULT_MAX_BUFFERED; + const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + let promptUse = options.promptUse; + + const pending: IngestEvent[] = []; + let droppedCount = 0; + let closed = false; + let warned = false; + + const headers: Record = { 'content-type': 'application/json' }; + if (token) headers.authorization = `Bearer ${token}`; + + const warnOnce = (what: string, error: unknown) => { + if (warned) return; + warned = true; + const reason = error instanceof Error ? error.message : String(error); + console.warn(`[lousho] observe: ${what} (${reason}); events are counted in 'dropped' and the run is unaffected.`); + }; + + const push = (event: IngestEvent) => { + if (pending.length >= maxBuffered) { + droppedCount++; + warnOnce(`buffer is full (${maxBuffered} events); dropping`, new SDKError('observe buffer full', 'LOUSHO_REMOTE_REQUEST_FAILED', { appendHelp: false })); + return; + } + pending.push(event); + if (pending.length >= flushSize) void flush(requestTimeoutMs); + }; + + const enqueue = (event: IngestEvent) => { + // A span ending after close() has no flush left to catch it - count it as dropped. + if (closed) { + droppedCount++; + return; + } + push(event); + }; + + const post = async (events: IngestEvent[], timeoutMs: number): Promise => { + const res = await fetchFn(`${url}/api/v1/ingest`, { + method: 'POST', + headers, + body: JSON.stringify({ batch: events }), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) { + throw new SDKError(`ingest answered ${res.status}`, 'LOUSHO_REMOTE_REQUEST_FAILED', { appendHelp: false }); + } + }; + + // Flushes are serialized on a chain: a fire-and-forget size flush and the + // exit flush can overlap, and close() must not resolve while its own POST + // is still in flight. + let chain: Promise = Promise.resolve(); + const flush = (timeoutMs: number): Promise => { + const run = chain.then(async () => { + if (pending.length === 0) return; + const batch = pending.splice(0, pending.length); + try { + await post(batch, timeoutMs); + } catch (error) { + // Drop-and-count by contract: a missing daemon must never slow or fail a run. + droppedCount += batch.length; + warnOnce(`could not reach the observe daemon at ${url}`, error); + } + }); + chain = run; + return run; + }; + + const heartbeat = (): void => { + if (heartbeatIntervalMs <= 0) return; + enqueue({ + type: 'agent.heartbeat', + data: { + agent_id: identity.agentId, + project, + name: agentName, + version, + ts: now(), + heartbeat_interval_ms: heartbeatIntervalMs, + ...(hostName() !== undefined ? { hostname: hostName() } : {}), + ...(typeof process !== 'undefined' && typeof process.pid === 'number' ? { pid: process.pid } : {}), + }, + }); + }; + + /** span.id -> trace_id; a span whose parent is unknown roots a trace of its own. */ + const targets = new Map(); + + const timers: ReturnType[] = []; + if (flushIntervalMs > 0) { + const t = setInterval(() => void flush(requestTimeoutMs), flushIntervalMs); + t.unref?.(); + timers.push(t); + } + if (heartbeatIntervalMs > 0) { + heartbeat(); // register at once so the fleet registry sees the agent on start, not 60s in + const t = setInterval(heartbeat, heartbeatIntervalMs); + t.unref?.(); + timers.push(t); + } + + const close = async (): Promise => { + if (closed) return; + closed = true; + for (const t of timers) clearInterval(t); + for (const [signal, handler] of exitHooks) { + try { + process.off(signal, handler); + } catch { + // no process.off in this runtime + } + } + // Graceful exit: say goodbye, then flush with the ~2s cap. push() directly so a + // full buffer can't eat the deregister - one pending event drops instead. + if (pending.length >= maxBuffered) pending.splice(0, 1); + push({ type: 'agent.deregister', data: { agent_id: identity.agentId, ts: now() } }); + await flush(EXIT_FLUSH_TIMEOUT_MS); + }; + + const exitHooks: Array<[string, () => void]> = []; + if (options.registerExitHooks !== false && typeof process !== 'undefined' && typeof process.once === 'function') { + // 'beforeExit' fires while async work still runs; the in-flight POST keeps the loop + // alive until it settles (the ~2s cap bounds the wait). Signals get an explicit + // exit afterwards because installing a handler replaces the default kill. + const onBeforeExit = () => void close(); + const onSignal = (code: number) => () => { + void close().finally(() => process.exit(code)); + }; + const hooks: Array<[NodeJS.Signals | 'beforeExit', () => void]> = [ + ['beforeExit', onBeforeExit], + ['SIGINT', onSignal(130)], + ['SIGTERM', onSignal(143)], + ]; + for (const [signal, handler] of hooks) { + process.once(signal as NodeJS.Signals, handler); + exitHooks.push([signal, handler]); + } + } + + return { + get dropped() { + return droppedCount; + }, + agentId: identity.agentId, + sessionId: fallbackSessionId, + get promptUse() { + return promptUse; + }, + set promptUse(provider: PromptUseProvider | undefined) { + promptUse = provider; + }, + onSpanStart(span) { + const parent = span.parentId !== undefined ? targets.get(span.parentId) : undefined; + targets.set(span.id, parent ?? span.id); + // A parentless span is a run's root `invoke_agent`: one `prompt_use` + // per run, keyed by the root span's id (the run's obs_id). Sub-agent + // runs arrive with parentSpanId set and never re-emit it. + if (span.parentId === undefined && promptUse !== undefined) { + const use = promptUse(); + if (use !== undefined) { + enqueue({ + type: 'prompt_use', + data: { obs_id: span.id, prompt_id: use.prompt_id, ...(use.version !== undefined ? { version: use.version } : {}) }, + }); + } + } + }, + onSpanEnd(span) { + const traceId = targets.get(span.id) ?? span.id; + enqueue({ type: 'observation', data: observationOf(span, traceId, identity.agentId, fallbackSessionId, now) }); + if (traceId !== span.id) return; + // The root ended: forget every span of this trace so the map does not grow forever. + for (const [id, tid] of targets) if (tid === traceId) targets.delete(id); + }, + close, + }; +} + +/* ------------------------------------------------------- env wiring + fanout */ + +/** What a fan-out resolves to, so re-composing does not double-deliver (see below). */ +const composedParts = new WeakMap(); + +/** + * Fan several exporters out to one: every span goes to each. Composed + * exporters remember their parts, so composing `fanOut([a, b])` with `b` + * again yields `a, b` once - `createAgent` composing over an exporter that + * `AgentExecutor.execute` already env-composed must not deliver twice. An + * inner exporter's throw is reported (through withSpan's once-warning) but + * never keeps the remaining exporters from seeing the span. + */ +export function fanOutExporters(exporters: readonly (TraceExporter | undefined | null)[]): TraceExporter | undefined { + const flat: TraceExporter[] = []; + const seen = new Set(); + for (const exporter of exporters) { + for (const inner of exporter ? (composedParts.get(exporter) ?? [exporter]) : []) { + if (!seen.has(inner)) { + seen.add(inner); + flat.push(inner); + } + } + } + if (flat.length === 0) return undefined; + if (flat.length === 1) return flat[0]; + const fan: TraceExporter = { + onSpanStart(span) { + let first: unknown; + for (const exporter of flat) { + try { + exporter.onSpanStart(span); + } catch (error) { + first ??= error; + } + } + if (first !== undefined) throw first; + }, + onSpanEnd(span) { + let first: unknown; + for (const exporter of flat) { + try { + exporter.onSpanEnd(span); + } catch (error) { + first ??= error; + } + } + if (first !== undefined) throw first; + }, + }; + composedParts.set(fan, flat); + return fan; +} + +const envReporters = new Map(); +/** Every reporter {@link observeExporterFromEnv} made - so a composed exporter can be checked for an env member. */ +const envReporterSet = new WeakSet(); + +/** + * The collector base URL the env points at: `LOUSHO_OBSERVE=1`/`true` → + * `http://127.0.0.1:$LOUSHO_OBSERVE_PORT(7777)`, a URL → itself, unset/`0`/ + * `false` → `undefined`. Shared by the reporter, the prompt fetch and the + * hub client - they must all agree on where the daemon lives. + */ +export function observeUrlFromEnv( + env: NodeJS.ProcessEnv | undefined = typeof process !== 'undefined' ? process.env : undefined +): string | undefined { + const flag = env?.LOUSHO_OBSERVE?.trim(); + if (!flag || flag === '0' || flag.toLowerCase() === 'false' || env === undefined) return undefined; + return flag === '1' || flag.toLowerCase() === 'true' + ? `http://127.0.0.1:${env.LOUSHO_OBSERVE_PORT || String(OBSERVE_DEFAULT_PORT)}` + : flag; +} + +/** + * The `LOUSHO_OBSERVE`-driven reporter, memoized per (agent name, project, + * url): every `execute()` call resolves the same instance, so an agent never + * heartbeats twice. `undefined` when the env is unset/`0`/`false`. + * + * `promptUse` is the registry prompt the agent resolved (`createAgent({ + * prompt })`/`LOUSHO_PROMPT`): it is set on the shared reporter, so the last + * agent built under one name wins. + * + * Pass `env` only in tests. + */ +export function observeExporterFromEnv( + options: { agentName?: string; promptUse?: PromptUseProvider; env?: NodeJS.ProcessEnv } = {} +): ObserveReporter | undefined { + const env = options.env ?? (typeof process !== 'undefined' ? process.env : undefined); + const url = observeUrlFromEnv(env); + if (url === undefined) return undefined; + const name = options.agentName ?? 'agent'; + const project = env?.LOUSHO_OBSERVE_PROJECT; + const key = `${name}${project ?? ''}${url}`; + let reporter = envReporters.get(key); + if (!reporter) { + reporter = observeExporter({ + url, + agentName: name, + ...(project ? { project } : {}), + ...(env?.LOUSHO_OBSERVE_TOKEN ? { token: env.LOUSHO_OBSERVE_TOKEN } : {}), + }); + envReporters.set(key, reporter); + envReporterSet.add(reporter); + } + if (options.promptUse !== undefined) reporter.promptUse = options.promptUse; + return reporter; +} + +/** The parts `explicit` flattens to (itself when it was never composed). */ +function exporterParts(explicit: TraceExporter): readonly TraceExporter[] { + return composedParts.get(explicit) ?? [explicit]; +} + +/** + * The exporter a run reports with: the caller's, the env's, or both fanned + * out. When `explicit` already carries an env reporter - a delegated + * sub-agent inherits its lead's exporter - the env step is skipped, so one + * process reports under ONE agent identity and child spans are not + * double-posted stamped with two agent_ids. + * + * `options.promptUse` is the registry prompt the agent resolved; it lands on + * the env reporter (the only path to the daemon's ingest API). + */ +export function withObserveExporter( + explicit: TraceExporter | undefined, + agentName: string | undefined, + options: { promptUse?: PromptUseProvider } = {} +): TraceExporter | undefined { + if (explicit && exporterParts(explicit).some((part) => envReporterSet.has(part))) return explicit; + return fanOutExporters([explicit, observeExporterFromEnv({ agentName, ...(options.promptUse ? { promptUse: options.promptUse } : {}) })]); +} diff --git a/src/observe/routes/agents.ts b/src/observe/routes/agents.ts new file mode 100644 index 00000000..5bc58fa2 --- /dev/null +++ b/src/observe/routes/agents.ts @@ -0,0 +1,52 @@ +/** + * `GET /api/v1/agents` - the fleet registry rows with Monitoring columns: + * sweep-owned `status` (heartbeat freshness: healthy/stale/down, or 'stopped' + * from a graceful `agent.deregister`) plus `runs_today`/`error_rate_today` + * from the day rollups - the same UTC bucket ingest recomputes, so this adds + * no scan of the observations table. + */ + +import type { SqlDatabase } from '../../storage/sqlite/driver'; +import { listAgents } from '../store'; +import { sendJson, type Route } from './httpKit'; + +const DAY_MS = 86_400_000; + +/** Per-agent run/error counts of today's UTC day rollup bucket. */ +function todayStats(db: SqlDatabase, now: number): Map { + const dayStart = now - (now % DAY_MS); + const stats = new Map(); + for (const row of db + .prepare("SELECT agent_id, metric, value FROM rollups WHERE granularity = 'day' AND bucket_start = ?") + .all(dayStart)) { + const entry = stats.get(String(row.agent_id)) ?? { runs: 0, errors: 0 }; + if (row.metric === 'runs') entry.runs += Number(row.value); + else if (row.metric === 'errors') entry.errors += Number(row.value); + stats.set(String(row.agent_id), entry); + } + return stats; +} + +export const agentsRoutes: readonly Route[] = [ + { + method: 'GET', + path: '/api/v1/agents', + handle: ({ db, res, query }) => { + const now = Date.now(); + let agents = listAgents(db, now); + const project = query.get('project'); + if (project !== null) agents = agents.filter((agent) => agent.project === project); + const stats = todayStats(db, now); + sendJson(res, 200, { + agents: agents.map((agent) => { + const today = stats.get(agent.agent_id) ?? { runs: 0, errors: 0 }; + return { + ...agent, + runs_today: today.runs, + error_rate_today: today.runs > 0 ? today.errors / today.runs : 0, + }; + }), + }); + }, + }, +]; diff --git a/src/observe/routes/httpKit.ts b/src/observe/routes/httpKit.ts new file mode 100644 index 00000000..e598fb70 --- /dev/null +++ b/src/observe/routes/httpKit.ts @@ -0,0 +1,96 @@ +/** + * The route kit the daemon's feature files share: the {@link Route}/ + * {@link RouteContext} contract, the bounded-body error, and the small + * response/param helpers. It lives beside `index.ts` rather than in it so + * feature files import it without closing an import cycle through the + * registry (which imports every feature file). + */ + +import type * as http from 'node:http'; +import { z } from 'zod'; +import { SDKError } from '../../execution/errors'; +import type { SqlDatabase } from '../../storage/sqlite/driver'; +import { sendJson } from '../../server/sendJson'; + +export { sendJson }; + +/** What a handler gets: the request, the matched `:params`, the query, the db, and a bounded body reader. */ +export interface RouteContext { + db: SqlDatabase; + req: http.IncomingMessage; + res: http.ServerResponse; + params: Record; + query: URLSearchParams; + /** + * The request body as text, at most `limitBytes`; a body over the limit + * throws {@link BodyTooLargeError} (the daemon answers `413`). Throws + * `LOUSHO_REQUEST_INVALID` on a second call. + */ + readBody(limitBytes: number): Promise; +} + +export interface Route { + method: 'GET' | 'POST' | 'PUT' | 'DELETE'; + /** Literal path template; `:name` captures one path segment. */ + path: string; + handle(ctx: RouteContext): void | Promise; +} + +/** Thrown by `ctx.readBody` when the body exceeds the byte cap; the daemon maps it to 413. */ +export class BodyTooLargeError extends Error { + override readonly name = 'BodyTooLargeError'; +} + +/** A `400`-shaped answer in the API's `{ error, code }` body style. */ +export function badRequest(res: http.ServerResponse, message: string): void { + sendJson(res, 400, { error: message, code: 'LOUSHO_REQUEST_INVALID' }); +} + +/** A query param as a bounded positive integer (`fallback` when absent). */ +export function intParam( + query: URLSearchParams, + name: string, + fallback: number, + bounds: { min: number; max: number } +): number { + const raw = query.get(name); + if (raw === null) return fallback; + const value = Number(raw); + if (!Number.isInteger(value)) return fallback; + return Math.min(bounds.max, Math.max(bounds.min, value)); +} + +/** A query param as epoch milliseconds, or `undefined`. */ +export function timeParam(query: URLSearchParams, name: string): number | undefined { + const raw = query.get(name); + if (raw === null) return undefined; + const value = Number(raw); + return Number.isFinite(value) ? value : undefined; +} + +/** Rejects a malformed JSON body with the envelope-parse code. */ +export function parseJsonBody(text: string): unknown { + try { + return JSON.parse(text) as unknown; + } catch { + throw new SDKError('The request body is not valid JSON.', 'LOUSHO_REQUEST_INVALID', { appendHelp: false }); + } +} + +/** + * `parseJsonBody` plus a zod check: the JSON parse and the schema failure + * both throw `LOUSHO_REQUEST_INVALID`, the schema failure naming `what` + * (`'hub publish'`, `'prompt publish'`) and the first issue. + */ +export function requireJsonBody(schema: S, text: string, what: string): z.infer { + const parsed = schema.safeParse(parseJsonBody(text)); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + throw new SDKError( + `Invalid ${what} body: ${issue ? `${issue.path.join('.') || 'body'}: ${issue.message}` : 'bad shape'}.`, + 'LOUSHO_REQUEST_INVALID', + { appendHelp: false } + ); + } + return parsed.data; +} diff --git a/src/observe/routes/hub.ts b/src/observe/routes/hub.ts new file mode 100644 index 00000000..0516dd9c --- /dev/null +++ b/src/observe/routes/hub.ts @@ -0,0 +1,135 @@ +/** + * The Context Hub (spec 0001-prompts-context-hub): namespaced shared memory + * between agents on one daemon. + * + * - `POST /api/v1/hub` publishes `{agent_id, namespace, key, value, scope?, + * ttl?}`. `agent_id` is the caller's reporter identity - the threat model + * trusts loopback, so it is sent in the body, not derived from auth. One + * entry per (namespace, agent_id, key): `entry_id` is their sha256, so + * republishing a key updates the row in place (a retried publish is + * idempotent). `value` is stored verbatim when a string, else JSON-encoded; + * `ttl` is a lifetime in milliseconds. + * - `GET /api/v1/hub?agent_id&namespace&scope&key&limit` queries. Scope is + * enforced HERE, daemon side, against the caller's `agent_id` (required): + * `private` returns only the caller's own private entries, `fleet` returns + * every agent's fleet entries, and no `scope` returns both - a private + * entry of another agent can never be read through this endpoint (the + * scope is organizational, not a security boundary, but the filter is not + * the client's job). Entries past their `ttl` are filtered at read. + */ + +import { createHash } from 'node:crypto'; +import { z } from 'zod'; +import { intParam, requireJsonBody, sendJson, type Route } from './httpKit'; + +const HUB_SCOPES = ['private', 'fleet'] as const; + +const publishBody = z.object({ + agent_id: z.string().min(1), + namespace: z.string().min(1).max(200), + key: z.string().min(1).max(500), + /** Any JSON value (or a plain string, stored as-is). */ + value: z.unknown(), + scope: z.enum(HUB_SCOPES).default('private'), + /** Lifetime in milliseconds; absent means the entry does not expire. */ + ttl: z.number().int().positive().optional(), +}); + +interface HubEntryRow { + entry_id: string; + namespace: string; + agent_id: string | null; + key: string; + value: string | null; + scope: string; + created_at: number; + ttl: number | null; + expires_at: number | null; +} + +function entryRow(row: Record): HubEntryRow { + const createdAt = Number(row.created_at); + const ttl = row.ttl === null || row.ttl === undefined ? null : Number(row.ttl); + return { + entry_id: String(row.entry_id), + namespace: String(row.namespace), + agent_id: (row.agent_id as string | null) ?? null, + key: String(row.key), + value: (row.value as string | null) ?? null, + scope: String(row.scope), + created_at: createdAt, + ttl, + expires_at: ttl === null ? null : createdAt + ttl, + }; +} + +/** One row per (namespace, agent_id, key): republishing the same key updates value/scope/ttl in place. */ +const ENTRY_UPSERT = ` + INSERT INTO hub_entries (entry_id, namespace, agent_id, "key", value, scope, created_at, ttl) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(entry_id) DO UPDATE SET + value = excluded.value, + scope = excluded.scope, + created_at = excluded.created_at, + ttl = excluded.ttl +`; + +export const hubRoutes: readonly Route[] = [ + { + method: 'POST', + path: '/api/v1/hub', + handle: async ({ db, res, readBody }) => { + const { agent_id, namespace, key, value, scope, ttl } = requireJsonBody( + publishBody, + await readBody(1024 * 1024), + 'hub publish' + ); + const stored = typeof value === 'string' ? value : JSON.stringify(value ?? null); + // JSON-encoded parts so ('a b','c') and ('a','b c') hash differently. + const entryId = createHash('sha256').update(JSON.stringify([namespace, agent_id, key])).digest('hex'); + const existed = db.prepare('SELECT entry_id FROM hub_entries WHERE entry_id = ?').get(entryId) !== undefined; + db.prepare(ENTRY_UPSERT).run(entryId, namespace, agent_id, key, stored, scope, Date.now(), ttl ?? null); + const row = db.prepare('SELECT * FROM hub_entries WHERE entry_id = ?').get(entryId); + sendJson(res, existed ? 200 : 201, { entry: row ? entryRow(row) : undefined, created: !existed }); + }, + }, + { + method: 'GET', + path: '/api/v1/hub', + handle: ({ db, res, query }) => { + const agentId = query.get('agent_id'); + if (!agentId) { + return sendJson(res, 400, { error: `The 'agent_id' query param is required; scope is enforced against the caller's identity.`, code: 'LOUSHO_REQUEST_INVALID' }); + } + const scope = query.get('scope'); + if (scope !== null && scope !== 'private' && scope !== 'fleet') { + return sendJson(res, 400, { error: `scope must be 'private' or 'fleet', got '${scope}'.`, code: 'LOUSHO_REQUEST_INVALID' }); + } + const params: (string | number)[] = []; + let where = '(ttl IS NULL OR created_at + ttl > ?)'; + params.push(Date.now()); + if (scope === 'private') { + where += ` AND scope = 'private' AND agent_id = ?`; + params.push(agentId); + } else if (scope === 'fleet') { + where += ` AND scope = 'fleet'`; + } else { + where += ` AND (scope = 'fleet' OR (scope = 'private' AND agent_id = ?))`; + params.push(agentId); + } + const namespace = query.get('namespace'); + if (namespace !== null) { + where += ' AND namespace = ?'; + params.push(namespace); + } + const key = query.get('key'); + if (key !== null) { + where += ' AND "key" = ?'; + params.push(key); + } + const limit = intParam(query, 'limit', 100, { min: 1, max: 1000 }); + const rows = db.prepare(`SELECT * FROM hub_entries WHERE ${where} ORDER BY created_at DESC LIMIT ?`).all(...params, limit); + sendJson(res, 200, { entries: rows.map(entryRow) }); + }, + }, +]; diff --git a/src/observe/routes/index.test.ts b/src/observe/routes/index.test.ts new file mode 100644 index 00000000..57bdbbe6 --- /dev/null +++ b/src/observe/routes/index.test.ts @@ -0,0 +1,28 @@ +/** + * The segment matcher behind every daemon route: literal paths match, + * `:param` segments capture one decoded segment, and method, segment count + * or a literal miss return `undefined`. + */ +import { describe, expect, it } from 'vitest'; +import { matchRoute } from './index'; + +describe('matchRoute', () => { + it('matches a literal path', () => { + const hit = matchRoute('GET', '/health'); + expect(hit?.route.path).toBe('/health'); + expect(hit?.params).toEqual({}); + }); + + it('captures :param segments, URI-decoded', () => { + const hit = matchRoute('GET', '/api/v1/traces/abc%2Fdef'); + expect(hit?.route.path).toBe('/api/v1/traces/:id'); + expect(hit?.params).toEqual({ id: 'abc/def' }); + }); + + it('misses on a different method, a different segment count, or a literal mismatch', () => { + expect(matchRoute('DELETE', '/api/v1/traces/x')).toBeUndefined(); + expect(matchRoute('GET', '/api/v1/traces/x/extra')).toBeUndefined(); + expect(matchRoute('GET', '/api/v2/traces')).toBeUndefined(); + expect(matchRoute('GET', '/api/v1/nope')).toBeUndefined(); + }); +}); diff --git a/src/observe/routes/index.ts b/src/observe/routes/index.ts new file mode 100644 index 00000000..6a652210 --- /dev/null +++ b/src/observe/routes/index.ts @@ -0,0 +1,69 @@ +/** + * The daemon's route registry. One file per feature area (ingest, traces, + * insights, agents, scores, status, prompts, hub) exports a `routes` array; this index + * concatenates them - so a later child spec (prompts, hub, alerts) adds a + * file here and never touches the daemon's dispatch internals. + * + * A route is `{ method, path, handler }` where `path` is a literal template + * with `:param` segments (`/api/v1/traces/:id`). No framework: the API is a + * dozen endpoints, so a segment matcher is enough and keeps `node:http` the + * only server dependency. + */ + +import { agentsRoutes } from './agents'; +import { hubRoutes } from './hub'; +import { ingestRoutes } from './ingest'; +import { insightsRoutes } from './insights'; +import { promptsRoutes } from './prompts'; +import { scoresRoutes } from './scores'; +import { statusRoutes } from './status'; +import { tracesRoutes } from './traces'; +import type { Route } from './httpKit'; + +// The shared route contract and helpers live in httpKit.ts; feature files +// import it directly so none of them has to close an import cycle through +// this registry. The pieces the daemon needs stay reachable from here too. +export { BodyTooLargeError, sendJson } from './httpKit'; +export type { RouteContext } from './httpKit'; + +/** Every registered route, feature files in order. Later children append their own import + spread. */ +const routes: readonly Route[] = [ + ...statusRoutes, + ...ingestRoutes, + ...tracesRoutes, + ...insightsRoutes, + ...agentsRoutes, + ...scoresRoutes, + ...promptsRoutes, + ...hubRoutes, +]; + +interface CompiledRoute { + route: Route; + segments: string[]; +} + +const compiled: CompiledRoute[] = routes.map((route) => ({ route, segments: route.path.split('/').filter(Boolean) })); + +/** Find the route for `method`/`pathname`; `params` gets the `:name` captures. `undefined` when none matches. */ +export function matchRoute( + method: string, + pathname: string +): { route: Route; params: Record } | undefined { + const parts = pathname.split('/').filter(Boolean); + for (const { route, segments } of compiled) { + if (route.method !== method || segments.length !== parts.length) continue; + const params: Record = {}; + let match = true; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]; + if (segment.startsWith(':')) params[segment.slice(1)] = decodeURIComponent(parts[i]); + else if (segment !== parts[i]) { + match = false; + break; + } + } + if (match) return { route, params }; + } + return undefined; +} diff --git a/src/observe/routes/ingest.ts b/src/observe/routes/ingest.ts new file mode 100644 index 00000000..b5e5b7d2 --- /dev/null +++ b/src/observe/routes/ingest.ts @@ -0,0 +1,27 @@ +/** + * `POST /api/v1/ingest` - the only write endpoint, and the contract a hosted + * collector implements later. Validates the envelope (`400` names every bad + * `batch[i]` path), enforces the ~5MB body cap (`413` via the bounded + * `readBody`), then runs the whole batch - upserts plus rollup recompute - in + * one transaction and acks `202` fast. + */ + +import { OBSERVE_MAX_BODY_BYTES, parseIngestEnvelope } from '../envelope'; +import { ingestBatch } from '../store'; +import { sendJson, parseJsonBody, type Route } from './httpKit'; + +export const ingestRoutes: readonly Route[] = [ + { + method: 'POST', + path: '/api/v1/ingest', + handle: async (ctx) => { + const body = await ctx.readBody(OBSERVE_MAX_BODY_BYTES); + const parsed = parseIngestEnvelope(parseJsonBody(body)); + if (!parsed.ok) { + return sendJson(ctx.res, 400, { error: `Invalid ingest envelope: ${parsed.issues.join('; ')}`, code: 'LOUSHO_REQUEST_INVALID' }); + } + const result = ingestBatch(ctx.db, parsed.envelope.batch); + sendJson(ctx.res, 202, { received: result.received }); + }, + }, +]; diff --git a/src/observe/routes/insights.ts b/src/observe/routes/insights.ts new file mode 100644 index 00000000..d4f1744f --- /dev/null +++ b/src/observe/routes/insights.ts @@ -0,0 +1,29 @@ +/** + * `GET /api/v1/insights` - per-agent aggregate cards plus the rollup time + * series the Insights charts draw. Aggregates come from `rollups` (runs/day, + * cost, error rate, tokens); latency percentiles are the sanctioned + * drilldown on `observations`. Window: `days` (default 7) or explicit + * `since`/`until` epoch-ms. + */ + +import { insightsAgents, rollupSeries, type RollupGranularity } from '../store'; +import { intParam, sendJson, timeParam, type Route } from './httpKit'; + +const DAY_MS = 86_400_000; + +export const insightsRoutes: readonly Route[] = [ + { + method: 'GET', + path: '/api/v1/insights', + handle: ({ db, res, query }) => { + const until = timeParam(query, 'until') ?? Date.now(); + const days = intParam(query, 'days', 7, { min: 1, max: 365 }); + const since = timeParam(query, 'since') ?? until - days * DAY_MS; + const agentId = query.get('agent_id') ?? undefined; + const granularity = (query.get('granularity') ?? 'hour') as RollupGranularity; + const agents = insightsAgents(db, { since, until }).filter((a) => agentId === undefined || a.agent_id === agentId); + const series = rollupSeries(db, { granularity: granularity === 'day' ? 'day' : 'hour', agent_id: agentId, since, until }); + sendJson(res, 200, { window: { since, until }, agents, series }); + }, + }, +]; diff --git a/src/observe/routes/prompts.ts b/src/observe/routes/prompts.ts new file mode 100644 index 00000000..b304bd18 --- /dev/null +++ b/src/observe/routes/prompts.ts @@ -0,0 +1,179 @@ +/** + * The prompt registry (spec 0001-prompts-context-hub). + * + * - `POST /api/v1/prompts` publishes `{name, text, labels?, variables?}` as a + * new version: `version` auto-increments per name and `hash` (sha256 of the + * text) dedupes a republish identical to the current latest - same hash, + * same labels and variables answers `200` with the existing row instead of + * minting a version. A republish of an OLDER text still creates a version + * (a rollback is a real event in the history). + * - `GET /api/v1/prompts` lists one row per name (latest version, version + * count, run count) - Studio's registry table. + * - `GET /api/v1/prompts/:name` is the version history, text included so a + * Studio panel (or a diff) can compare versions without N fetches. + * - `GET /api/v1/prompts/:name/:version` resolves one version - `:version` is + * an integer or `latest`. `latest` is resolved DAEMON SIDE on purpose: + * every agent that asks in the same moment gets the same row, which is the + * whole point of the movable label. + * + * Names go in the URL path, so they may not contain `/`, `@` or whitespace - + * the same constraint `parsePromptRef` applies on the client. + */ + +import { createHash } from 'node:crypto'; +import { z } from 'zod'; +import type { SqlDatabase } from '../../storage/sqlite/driver'; +import { intParam, requireJsonBody, sendJson, type Route } from './httpKit'; + +/** A prompt name is a URL path segment and the left half of a `name@version` ref. */ +const NAME_PATTERN = /^[^\s@/]{1,200}$/; + +const publishBody = z.object({ + name: z.string().regex(NAME_PATTERN, 'a name is a non-empty path segment without whitespace or @ (e.g. "mira", "support-v2")'), + text: z.string().min(1), + /** Free-form labels (e.g. ["prod"]); stored verbatim as JSON. */ + labels: z.array(z.string()).or(z.record(z.string(), z.string())).optional(), + /** Variable names the text interpolates (templating is a later spec; stored verbatim). */ + variables: z.array(z.string()).optional(), +}); + +interface PromptRow { + prompt_id: string; + name: string; + version: number; + labels: unknown; + variables: unknown; + hash: string | null; + text: string | null; + created_at: number; +} + +const parseJson = (value: unknown): unknown => { + if (typeof value !== 'string') return value ?? null; + try { + return JSON.parse(value) as unknown; + } catch { + return value; + } +}; + +function promptRow(row: Record): PromptRow { + return { + prompt_id: String(row.prompt_id), + name: String(row.name), + version: Number(row.version), + labels: parseJson(row.labels), + variables: parseJson(row.variables), + hash: (row.hash as string | null) ?? null, + text: (row.text as string | null) ?? null, + created_at: Number(row.created_at), + }; +} + +const sha256 = (text: string): string => createHash('sha256').update(text).digest('hex'); + +/** `prompt_id` is deterministic (`sha256(name@version)`): a retried publish of the same version upserts, never duplicates. */ +const PROMPT_UPSERT = ` + INSERT INTO prompts (prompt_id, name, version, labels, text, variables, hash, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(prompt_id) DO UPDATE SET + labels = excluded.labels, + text = excluded.text, + variables = excluded.variables, + hash = excluded.hash +`; + +function latestRow(db: SqlDatabase, name: string): Record | undefined { + return db.prepare('SELECT * FROM prompts WHERE name = ? ORDER BY version DESC LIMIT 1').get(name); +} + +/** Identical republish = same text hash AND same labels/variables JSON as the latest version. */ +function isIdenticalTo(latest: Record, hash: string, labels: string | null, variables: string | null): boolean { + return latest.hash === hash && (latest.labels ?? null) === labels && (latest.variables ?? null) === variables; +} + +export const promptsRoutes: readonly Route[] = [ + { + method: 'POST', + path: '/api/v1/prompts', + handle: async ({ db, res, readBody }) => { + const { name, text, labels, variables } = requireJsonBody( + publishBody, + await readBody(1024 * 1024), + 'prompt publish' + ); + const hash = sha256(text); + const labelsJson = labels === undefined ? null : JSON.stringify(labels); + const variablesJson = variables === undefined ? null : JSON.stringify(variables); + const latest = latestRow(db, name); + if (latest && isIdenticalTo(latest, hash, labelsJson, variablesJson)) { + return sendJson(res, 200, { prompt: promptRow(latest), created: false }); + } + const version = latest === undefined ? 1 : Number(latest.version) + 1; + const promptId = sha256(`${name}@${version}`); + db.prepare(PROMPT_UPSERT).run(promptId, name, version, labelsJson, text, variablesJson, hash, Date.now()); + const row = db.prepare('SELECT * FROM prompts WHERE prompt_id = ?').get(promptId); + sendJson(res, 201, { + prompt: row + ? promptRow(row) + : { prompt_id: promptId, name, version, labels: labels ?? null, variables: variables ?? null, hash, text, created_at: Date.now() }, + created: true, + }); + }, + }, + { + method: 'GET', + path: '/api/v1/prompts', + handle: ({ db, res, query }) => { + const limit = intParam(query, 'limit', 200, { min: 1, max: 1000 }); + const name = query.get('name'); + const rows = db + .prepare( + `SELECT p.*, + (SELECT COUNT(*) FROM prompts v WHERE v.name = p.name) AS versions, + (SELECT COUNT(*) FROM prompt_uses u JOIN prompts q ON q.prompt_id = u.prompt_id WHERE q.name = p.name) AS uses + FROM prompts p + WHERE p.version = (SELECT MAX(version) FROM prompts m WHERE m.name = p.name) + AND (? IS NULL OR p.name = ?) + ORDER BY p.name LIMIT ?` + ) + .all(name, name, limit); + sendJson(res, 200, { + prompts: rows.map((row) => ({ ...promptRow(row), versions: Number(row.versions), uses: Number(row.uses) })), + }); + }, + }, + { + method: 'GET', + path: '/api/v1/prompts/:name', + handle: ({ db, res, params }) => { + const rows = db + .prepare( + `SELECT p.*, (SELECT COUNT(*) FROM prompt_uses u WHERE u.prompt_id = p.prompt_id) AS uses + FROM prompts p WHERE p.name = ? ORDER BY p.version DESC` + ) + .all(params.name); + if (rows.length === 0) { + return sendJson(res, 404, { error: `No prompt '${params.name}'.`, code: 'LOUSHO_REQUEST_INVALID' }); + } + sendJson(res, 200, { name: params.name, versions: rows.map((row) => ({ ...promptRow(row), uses: Number(row.uses) })) }); + }, + }, + { + method: 'GET', + path: '/api/v1/prompts/:name/:version', + handle: ({ db, res, params }) => { + if (params.version !== 'latest' && !/^\d+$/.test(params.version)) { + return sendJson(res, 400, { error: `Version '${params.version}' is not an integer or 'latest'.`, code: 'LOUSHO_REQUEST_INVALID' }); + } + const row = + params.version === 'latest' + ? latestRow(db, params.name) + : db.prepare('SELECT * FROM prompts WHERE name = ? AND version = ?').get(params.name, Number(params.version)); + if (row === undefined) { + return sendJson(res, 404, { error: `No prompt '${params.name}@${params.version}'.`, code: 'LOUSHO_REQUEST_INVALID' }); + } + sendJson(res, 200, { prompt: promptRow(row) }); + }, + }, +]; diff --git a/src/observe/routes/scores.test.ts b/src/observe/routes/scores.test.ts new file mode 100644 index 00000000..4079d535 --- /dev/null +++ b/src/observe/routes/scores.test.ts @@ -0,0 +1,166 @@ +/** + * `GET /api/v1/scores` read surface: row filters (trace, name, eval-name + * prefix, agent, source, time range) and the `aggregate=eval` pass-rate + * rollup the Studio eval-history view reads. + */ +import { describe, expect, it } from 'vitest'; +import type * as http from 'node:http'; +import type { IngestEvent } from '../envelope'; +import { ingestBatch, openDb, type ScoreRow } from '../store'; +import type { SqlDatabase } from '../../storage/sqlite/driver'; +// Imported through the route registry (production's load order): importing +// './scores' first would hit the index<->feature-file cycle with scoresRoutes +// still uninitialized. +import { matchRoute } from './index'; + +const T0 = 1_700_000_000_000; + +const score = (over: Record = {}): IngestEvent => ({ + type: 'score', + data: { score_id: `s${Math.random().toString(36).slice(2)}`, trace_id: 't1', name: 'eval-a/completed()', value: 1, source: 'eval', ts: T0, ...over }, +}); + +function db(): SqlDatabase { + return openDb(':memory:'); +} + +interface Captured { + status: number; + body: unknown; +} + +/** Calls the route's handler with a recording ServerResponse stand-in. */ +function get(database: SqlDatabase, query: string): Captured { + const captured: Captured = { status: 0, body: undefined }; + const res = { + writeHead(status: number) { + captured.status = status; + return res; + }, + end(chunk?: string) { + captured.body = chunk === undefined ? undefined : JSON.parse(chunk); + }, + } as unknown as http.ServerResponse; + const matched = matchRoute('GET', '/api/v1/scores'); + if (!matched) throw new Error('scores route not registered'); + matched.route.handle({ + db: database, + req: {} as http.IncomingMessage, + res, + params: matched.params, + query: new URLSearchParams(query), + readBody: () => Promise.reject(new Error('GET has no body')), + }); + return captured; +} + +const scoresOf = (captured: Captured): ScoreRow[] => (captured.body as { scores: ScoreRow[] }).scores; + +describe('GET /api/v1/scores filters', () => { + it('filters by trace_id, name, agent_id and source', () => { + const database = db(); + try { + ingestBatch(database, [ + score({ score_id: 'a', trace_id: 't1', name: 'eval-a/completed()', agent_id: 'ag1' }), + score({ score_id: 'b', trace_id: 't2', name: 'eval-a/tone', agent_id: 'ag1', source: 'judge' }), + score({ score_id: 'c', trace_id: 't1', name: 'eval-b/completed()', agent_id: 'ag2' }), + ]); + expect(scoresOf(get(database, 'trace_id=t1')).map((s) => s.score_id).sort()).toEqual(['a', 'c']); + expect(scoresOf(get(database, 'name=eval-a/tone')).map((s) => s.score_id)).toEqual(['b']); + expect(scoresOf(get(database, 'agent_id=ag1')).map((s) => s.score_id).sort()).toEqual(['a', 'b']); + expect(scoresOf(get(database, 'source=judge')).map((s) => s.score_id)).toEqual(['b']); + } finally { + database.close(); + } + }); + + it('matches the eval-name prefix with eval= (the row itself and its assertion family)', () => { + const database = db(); + try { + ingestBatch(database, [ + score({ score_id: 'a', name: 'refund/completed()' }), + score({ score_id: 'b', name: 'refund/tone' }), + score({ score_id: 'c', name: 'refund-extra/other' }), + score({ score_id: 'd', name: 'plain' }), + ]); + const rows = scoresOf(get(database, 'eval=refund')); + expect(rows.map((s) => s.score_id).sort()).toEqual(['a', 'b']); + expect(scoresOf(get(database, 'eval=plain')).map((s) => s.score_id)).toEqual(['d']); + } finally { + database.close(); + } + }); + + it('bounds by created_at with since/until and honours limit', () => { + const database = db(); + try { + ingestBatch(database, [ + score({ score_id: 'old', ts: T0 - 10_000 }), + score({ score_id: 'mid', ts: T0 }), + score({ score_id: 'new', ts: T0 + 10_000 }), + ]); + expect(scoresOf(get(database, `since=${T0}`)).map((s) => s.score_id).sort()).toEqual(['mid', 'new']); + expect(scoresOf(get(database, `until=${T0}`)).map((s) => s.score_id)).toEqual(['old']); + expect(scoresOf(get(database, 'limit=1'))).toHaveLength(1); + } finally { + database.close(); + } + }); +}); + +describe('GET /api/v1/scores?aggregate=eval', () => { + it('groups by eval name and reports pass-rate, mean value and last-seen', () => { + const database = db(); + try { + ingestBatch(database, [ + // refund: two passes (one skipped-style perfect score with a comment), one failure + score({ score_id: 'a', name: 'refund/completed()', value: 1, ts: T0 }), + score({ score_id: 'b', name: 'refund/tone', value: 0.9, source: 'judge', ts: T0 + 1000 }), + score({ score_id: 'c', name: 'refund/tone', value: 0.2, source: 'judge', comment: 'soft check failed', ts: T0 + 2000 }), + score({ score_id: 'd', name: 'refund/budget', value: 1, comment: 'skipped: no cost data', ts: T0 + 3000 }), + // greet: one failure only + score({ score_id: 'e', name: 'greet/completed()', value: 0, comment: 'completed() failed', ts: T0 + 4000 }), + ]); + const res = get(database, 'aggregate=eval'); + expect(res.status).toBe(200); + const { evals } = res.body as { evals: { name: string; count: number; passed: number; pass_rate: number; avg_value: number | null; last_ts: number }[] }; + expect(evals).toHaveLength(2); + const greet = evals.find((e) => e.name === 'greet'); + const refund = evals.find((e) => e.name === 'refund'); + expect(greet).toMatchObject({ count: 1, passed: 0, pass_rate: 0, last_ts: T0 + 4000 }); + expect(refund).toMatchObject({ count: 4, passed: 3, pass_rate: 0.75, last_ts: T0 + 3000 }); + expect(refund?.avg_value).toBeCloseTo((1 + 0.9 + 0.2 + 1) / 4); + // newest first + expect(evals[0].name).toBe('greet'); + } finally { + database.close(); + } + }); + + it('honours the same filters (agent, time range) as the row listing', () => { + const database = db(); + try { + ingestBatch(database, [ + score({ score_id: 'a', name: 'eval/a', agent_id: 'ag1', ts: T0 }), + score({ score_id: 'b', name: 'eval/b', agent_id: 'ag2', ts: T0 + 999_999 }), + ]); + const res = get(database, `aggregate=eval&agent_id=ag1`); + const { evals } = res.body as { evals: { name: string; count: number }[] }; + expect(evals).toEqual([{ name: 'eval', count: 1, passed: 1, pass_rate: 1, avg_value: 1, last_ts: T0 }]); + const bounded = get(database, `aggregate=eval&until=${T0 + 1}`); + expect((bounded.body as { evals: { count: number }[] }).evals[0].count).toBe(1); + } finally { + database.close(); + } + }); + + it('rejects a bad aggregate or source value with 400', () => { + const database = db(); + try { + expect(get(database, 'aggregate=bogus').status).toBe(400); + expect(get(database, 'source=bogus').status).toBe(400); + } finally { + database.close(); + } + }); +}); diff --git a/src/observe/routes/scores.ts b/src/observe/routes/scores.ts new file mode 100644 index 00000000..5145f490 --- /dev/null +++ b/src/observe/routes/scores.ts @@ -0,0 +1,46 @@ +/** + * `GET /api/v1/scores` - eval/judge/human score rows, filterable by trace, + * name, eval name prefix (`eval`, since score names are + * `/`), agent, source and created_at range. + * The evaluators child emits them through ingest; this is only the read + * surface the data plane owes Studio. + * + * `GET /api/v1/scores?aggregate=eval` returns the eval-history view's + * aggregate instead of rows: pass-rate, mean value and last-seen time per + * eval name (see store.scorePassRates). + */ + +import { SCORE_SOURCES } from '../envelope'; +import { listScores, scorePassRates, type ScoreFilter } from '../store'; +import { badRequest, intParam, sendJson, timeParam, type Route } from './httpKit'; + +export const scoresRoutes: readonly Route[] = [ + { + method: 'GET', + path: '/api/v1/scores', + handle: ({ db, res, query }) => { + const filter: ScoreFilter = { + trace_id: query.get('trace_id') ?? undefined, + name: query.get('name') ?? undefined, + eval: query.get('eval') ?? undefined, + agent_id: query.get('agent_id') ?? undefined, + source: query.get('source') ?? undefined, + since: timeParam(query, 'since'), + until: timeParam(query, 'until'), + limit: intParam(query, 'limit', 200, { min: 1, max: 1000 }), + }; + if (filter.source !== undefined && !(SCORE_SOURCES as readonly string[]).includes(filter.source)) { + return badRequest(res, `source must be one of ${SCORE_SOURCES.join(', ')}.`); + } + const aggregate = query.get('aggregate'); + if (aggregate === 'eval') { + return sendJson(res, 200, { evals: scorePassRates(db, filter) }); + } + if (aggregate !== null) { + return badRequest(res, "aggregate must be 'eval'."); + } + const { scores } = listScores(db, filter); + sendJson(res, 200, { scores }); + }, + }, +]; diff --git a/src/observe/routes/status.ts b/src/observe/routes/status.ts new file mode 100644 index 00000000..0195d801 --- /dev/null +++ b/src/observe/routes/status.ts @@ -0,0 +1,28 @@ +/** + * Health and status. `/health` is the port-collision marker: `lousho observe + * start` probes it on EADDRINUSE, and the `service` field is what tells a + * running collector apart from anything else that grabbed the port - it must + * stay unauthenticated for attach detection to work. `/api/v1/status` is the + * row-count surface `lousho observe status` prints. + */ + +import { VERSION } from '../../version'; +import { observeCounts } from '../store'; +import { sendJson, type Route } from './httpKit'; + +export const statusRoutes: readonly Route[] = [ + { + method: 'GET', + path: '/health', + handle: ({ res }) => { + sendJson(res, 200, { service: 'lousho-observe', ok: true, version: VERSION }); + }, + }, + { + method: 'GET', + path: '/api/v1/status', + handle: ({ db, res }) => { + sendJson(res, 200, { service: 'lousho-observe', ok: true, version: VERSION, counts: observeCounts(db) }); + }, + }, +]; diff --git a/src/observe/routes/traces.ts b/src/observe/routes/traces.ts new file mode 100644 index 00000000..7a12bd52 --- /dev/null +++ b/src/observe/routes/traces.ts @@ -0,0 +1,41 @@ +/** + * The Studio trace reads. `GET /api/v1/traces` is the paginated, filterable + * trace list (one row per `trace_id`, status from the root observation); + * `GET /api/v1/traces/:id` returns the trace's whole observation set ordered + * by start - flat rows the panel nests by `parent_obs_id` - plus its scores. + */ + +import { listTraces, traceDetail, type TraceFilter } from '../store'; +import { intParam, sendJson, timeParam, type Route } from './httpKit'; + +const LIST_LIMIT = { min: 1, max: 500 }; + +export const tracesRoutes: readonly Route[] = [ + { + method: 'GET', + path: '/api/v1/traces', + handle: ({ db, res, query }) => { + const filter: TraceFilter = { + agent_id: query.get('agent_id') ?? undefined, + project: query.get('project') ?? undefined, + status: query.get('status') ?? undefined, + session_id: query.get('session_id') ?? undefined, + since: timeParam(query, 'since'), + until: timeParam(query, 'until'), + limit: intParam(query, 'limit', 50, LIST_LIMIT), + offset: intParam(query, 'offset', 0, { min: 0, max: Number.MAX_SAFE_INTEGER }), + }; + const { traces, total } = listTraces(db, filter); + sendJson(res, 200, { traces, total, limit: filter.limit, offset: filter.offset }); + }, + }, + { + method: 'GET', + path: '/api/v1/traces/:id', + handle: ({ db, res, params }) => { + const detail = traceDetail(db, params.id); + if (!detail) return sendJson(res, 404, { error: `No trace '${params.id}'.`, code: 'LOUSHO_REQUEST_INVALID' }); + sendJson(res, 200, detail); + }, + }, +]; diff --git a/src/observe/schema.ts b/src/observe/schema.ts new file mode 100644 index 00000000..62969e04 --- /dev/null +++ b/src/observe/schema.ts @@ -0,0 +1,180 @@ +/** + * The collector database's schema. Like the session store (src/storage/sqlite), + * migrations are forward-only strings applied by the shared `migrate()` runner + * against `PRAGMA user_version` - append new entries, never edit shipped ones. + * + * The tables are the cross-child contract of + * docs/specs/0001-observability-platform/index.md: `observations` is the wide + * denormalized table every run event lands in (traces are virtual - the set of + * rows sharing one `trace_id`), `agents`/`sessions` are the registry, + * `scores`/`prompts`/`prompt_uses`/`hub_entries` belong to later children but + * are created here so the data plane's db is the only file the platform needs, + * `rollups` is a disposable derivation (never a second write path), and + * `tokens` holds hashed daemon keys. + * + * No FOREIGN KEY constraints: ingest is batched and order is not guaranteed + * (a `prompt_use` can land before its observation; scores can name traces + * emitted by another agent). Integrity comes from primary-key upserts, not + * from rejecting rows. + */ + +import type { SqlDatabase } from '../storage/sqlite/driver'; +import { migrate } from '../storage/sqlite/migrations'; + +/** + * Forward-only migrations for the observe database. `OBSERVE_MIGRATIONS[n]` + * upgrades a database from `user_version` n to n + 1. + */ +const OBSERVE_MIGRATIONS: readonly string[] = [ + ` + -- Fleet registry: one row per reporting agent. status is the last known + -- state ('healthy' on heartbeat, 'stopped' on deregister); monitoring + -- derives freshness from last_heartbeat at read/sweep time. + CREATE TABLE agents ( + agent_id TEXT PRIMARY KEY, + project TEXT, + name TEXT, + version TEXT, + first_seen INTEGER NOT NULL, + last_heartbeat INTEGER, + status TEXT NOT NULL DEFAULT 'healthy', + meta TEXT, + updated_at INTEGER NOT NULL + ); + + -- Conversation threads. Rows are derived lazily from observations that + -- carry a session_id (there is no session event type in the envelope). + CREATE TABLE sessions ( + session_id TEXT PRIMARY KEY, + agent_id TEXT, + started_at INTEGER NOT NULL, + ended_at INTEGER, + status TEXT NOT NULL DEFAULT 'active', + updated_at INTEGER NOT NULL + ); + CREATE INDEX sessions_agent ON sessions (agent_id); + + -- The wide table (Langfuse v4 shape): every finished span, generation, + -- tool call and point-in-time event, with trace-level fields flattened + -- onto the row. obs_id is the reporter's TraceLine.id; resends upsert + -- by key and updated_at gives last writer wins. + CREATE TABLE observations ( + obs_id TEXT PRIMARY KEY, + trace_id TEXT NOT NULL, + session_id TEXT, + agent_id TEXT, + parent_obs_id TEXT, + kind TEXT NOT NULL, + name TEXT, + start INTEGER NOT NULL, + end INTEGER, + status TEXT, + input TEXT, + output TEXT, + model TEXT, + usage_input_tokens REAL, + usage_output_tokens REAL, + usage_cached_tokens REAL, + usage_reasoning_tokens REAL, + cost_usd REAL, + cost_partial INTEGER, + usage_estimated INTEGER, + truncated INTEGER NOT NULL DEFAULT 0, + attrs TEXT, + updated_at INTEGER NOT NULL + ); + CREATE INDEX observations_trace ON observations (trace_id); + CREATE INDEX observations_session ON observations (session_id); + CREATE INDEX observations_agent ON observations (agent_id); + CREATE INDEX observations_start ON observations (start); + CREATE INDEX observations_parent ON observations (parent_obs_id); + + -- Eval results beside runs (evaluators child emits them; the data plane stores). + CREATE TABLE scores ( + score_id TEXT PRIMARY KEY, + trace_id TEXT NOT NULL, + obs_id TEXT, + name TEXT NOT NULL, + value REAL, + source TEXT NOT NULL, + comment TEXT, + agent_id TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX scores_trace ON scores (trace_id); + CREATE INDEX scores_name ON scores (name); + + -- The prompt registry (prompts-context-hub child): one row per published version. + CREATE TABLE prompts ( + prompt_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + version INTEGER NOT NULL, + labels TEXT, + text TEXT, + variables TEXT, + hash TEXT, + created_at INTEGER NOT NULL + ); + CREATE INDEX prompts_name_version ON prompts (name, version); + + -- Which prompt version a run's observation ran with. + CREATE TABLE prompt_uses ( + obs_id TEXT NOT NULL, + prompt_id TEXT NOT NULL, + version INTEGER, + created_at INTEGER NOT NULL, + PRIMARY KEY (obs_id, prompt_id) + ); + CREATE INDEX prompt_uses_prompt ON prompt_uses (prompt_id); + + -- Shared memory between agents (prompts-context-hub child). scope is + -- organizational, not a security boundary (the threat model trusts loopback). + CREATE TABLE hub_entries ( + entry_id TEXT PRIMARY KEY, + namespace TEXT NOT NULL, + agent_id TEXT, + "key" TEXT NOT NULL, + value TEXT, + scope TEXT NOT NULL DEFAULT 'private', + created_at INTEGER NOT NULL, + ttl INTEGER + ); + CREATE INDEX hub_entries_namespace ON hub_entries (namespace, scope, "key"); + + -- Insights aggregates, a disposable derivation of observations: + -- recomputed with INSERT ... SELECT ... GROUP BY inside the ingest + -- transaction (so a resent batch can never double count), and wholesale + -- by 'lousho observe rebuild-rollups'. Metrics per (bucket, agent): + -- 'runs', 'errors', 'cost_usd', 'input_tokens', 'output_tokens', + -- 'latency_ms' (sum of root-observation durations). + CREATE TABLE rollups ( + bucket_start INTEGER NOT NULL, + granularity TEXT NOT NULL, + agent_id TEXT NOT NULL, + metric TEXT NOT NULL, + value REAL NOT NULL, + PRIMARY KEY (bucket_start, granularity, agent_id, metric) + ); + CREATE INDEX rollups_agent ON rollups (agent_id, granularity, bucket_start); + + -- Daemon auth keys, hashed (sha256). Only checked when the daemon is + -- bound non-loopback or --require-token is set; loopback trusts the machine. + CREATE TABLE tokens ( + token_id TEXT PRIMARY KEY, + project TEXT, + key_hash TEXT NOT NULL UNIQUE, + created_at INTEGER NOT NULL, + revoked_at INTEGER + ); + `, +]; + +/** + * Bring `db` to the newest observe schema version. Thin wrapper over the + * shared runner so this module is the only place that knows which migration + * set the collector database uses. + */ +export function migrateObserve(db: SqlDatabase): number { + return migrate(db, OBSERVE_MIGRATIONS); +} diff --git a/src/observe/store.test.ts b/src/observe/store.test.ts new file mode 100644 index 00000000..03dcc7e6 --- /dev/null +++ b/src/observe/store.test.ts @@ -0,0 +1,262 @@ +/** + * The store's contract: idempotent upserts by primary key (a resent batch + * lands once and rollups never double count), rollup recompute correctness, + * truncation at the 64KB field cap, and token create/verify. + */ +import { describe, expect, it } from 'vitest'; +import type { SqlDatabase } from '../storage/sqlite/driver'; +import type { IngestEvent } from './envelope'; +import { + createToken, + hasTokens, + ingestBatch, + listAgents, + listTraces, + observeCounts, + openDb, + rebuildRollups, + rollupSeries, + traceDetail, + transaction, + verifyToken, +} from './store'; + +const T0 = 1_700_000_000_000; + +function db(): SqlDatabase { + return openDb(':memory:'); +} + +function obs(over: Partial['data']> = {}): IngestEvent { + return { + type: 'observation', + data: { obs_id: 'o1', trace_id: 't1', agent_id: 'a1', kind: 'span', name: 'invoke_agent a', start: T0, end: T0 + 100, status: 'ok', ...over }, + }; +} + +const heartbeat = (agentId: string, ts: number, project = 'proj'): IngestEvent => ({ + type: 'agent.heartbeat', + data: { agent_id: agentId, project, name: 'a', ts }, +}); + +describe('openDb', () => { + it('migrates an empty db to the observe schema', () => { + const database = db(); + try { + const version = database.prepare('PRAGMA user_version').get(); + expect(Number(version?.user_version)).toBeGreaterThan(0); + expect(database.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").get()).toBeDefined(); + expect(database.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='hub_entries'").get()).toBeDefined(); + expect(database.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='tokens'").get()).toBeDefined(); + } finally { + database.close(); + } + }); +}); + +describe('ingestBatch', () => { + it('upserts: a resent batch lands one row and never double-counts rollups', () => { + const database = db(); + try { + const batch = [heartbeat('a1', T0), obs()]; + ingestBatch(database, batch, T0); + ingestBatch(database, batch, T0 + 1); + const counts = observeCounts(database); + expect(counts.observations).toBe(1); + expect(counts.agents).toBe(1); + expect(rollupSeries(database, { agent_id: 'a1' }).filter((r) => r.metric === 'runs').every((r) => r.value === 1)).toBe(true); + } finally { + database.close(); + } + }); + + it('stores usage/cost verbatim including the partial and estimated flags', () => { + const database = db(); + try { + ingestBatch(database, [ + obs({ + usage_input_tokens: 11, + usage_output_tokens: 7, + usage_cached_tokens: 3, + usage_reasoning_tokens: 2, + cost_usd: 0.0042, + cost_partial: true, + usage_estimated: true, + model: 'm', + }), + ]); + const detail = traceDetail(database, 't1'); + const row = detail?.observations[0]; + expect(row?.usage_input_tokens).toBe(11); + expect(row?.usage_output_tokens).toBe(7); + expect(row?.usage_cached_tokens).toBe(3); + expect(row?.usage_reasoning_tokens).toBe(2); + expect(row?.cost_usd).toBe(0.0042); + expect(row?.cost_partial).toBe(true); + expect(row?.usage_estimated).toBe(true); + } finally { + database.close(); + } + }); + + it('truncates input/output at 64KB and sets the truncated flag', () => { + const database = db(); + try { + const big = 'x'.repeat(70 * 1024); + ingestBatch(database, [obs({ input: big, output: 'ok' })]); + const row = traceDetail(database, 't1')?.observations[0]; + expect(row?.input?.length).toBe(64 * 1024); + expect(row?.output).toBe('ok'); + expect(row?.truncated).toBe(true); + } finally { + database.close(); + } + }); + + it('keeps an already-flagged row truncated when the strings fit', () => { + const database = db(); + try { + ingestBatch(database, [obs({ input: 'small', truncated: true })]); + expect(traceDetail(database, 't1')?.observations[0]?.truncated).toBe(true); + } finally { + database.close(); + } + }); + + it('derives a session row from observations that carry a session_id', () => { + const database = db(); + try { + ingestBatch(database, [obs({ session_id: 's1' }), obs({ obs_id: 'o2', session_id: 's1', start: T0 + 50, end: T0 + 200 })]); + const session = database.prepare('SELECT * FROM sessions WHERE session_id = ?').get('s1'); + expect(session).toBeDefined(); + expect(Number(session?.started_at)).toBe(T0); + expect(Number(session?.ended_at)).toBe(T0 + 200); + } finally { + database.close(); + } + }); + + it('heartbeats upsert the agent row and keep the newest last_heartbeat', () => { + const database = db(); + try { + ingestBatch(database, [heartbeat('a1', T0)]); + ingestBatch(database, [heartbeat('a1', T0 - 5000)]); + const [agent] = listAgents(database, T0); + expect(agent.last_heartbeat).toBe(T0); + expect(agent.status).toBe('healthy'); + ingestBatch(database, [{ type: 'agent.deregister', data: { agent_id: 'a1', ts: T0 + 1 } }]); + expect(listAgents(database, T0)[0].status).toBe('stopped'); + } finally { + database.close(); + } + }); +}); + +describe('rollups', () => { + it('recomputes per-agent metrics for the touched buckets only', () => { + const database = db(); + try { + const otherHour = T0 + 4 * 3600_000; + ingestBatch(database, [ + obs(), + obs({ obs_id: 'o2', status: 'error', cost_usd: 0.5, usage_input_tokens: 10, usage_output_tokens: 4 }), + obs({ obs_id: 'o3', agent_id: 'a2' }), + obs({ obs_id: 'o4', start: otherHour, end: otherHour + 10 }), + // a child span (has a parent) is never a run + obs({ obs_id: 'o5', parent_obs_id: 'o1', kind: 'tool' }), + ]); + const hour = T0 - (T0 % 3600_000); + const a1 = rollupSeries(database, { agent_id: 'a1', granularity: 'hour' }); + const at = (bucket: number) => a1.filter((r) => r.bucket_start === bucket); + const metric = (bucket: number, m: string) => at(bucket).find((r) => r.metric === m)?.value; + expect(metric(hour, 'runs')).toBe(2); + expect(metric(hour, 'errors')).toBe(1); + expect(metric(hour, 'cost_usd')).toBe(0.5); + expect(metric(hour, 'input_tokens')).toBe(10); + expect(metric(hour, 'output_tokens')).toBe(4); + expect(metric(hour, 'latency_ms')).toBe(200); + expect(metric(otherHour - (otherHour % 3600_000), 'runs')).toBe(1); + expect(rollupSeries(database, { agent_id: 'a2', granularity: 'hour' }).find((r) => r.metric === 'runs')?.value).toBe(1); + // day granularity got the same runs + const day = T0 - (T0 % 86_400_000); + expect(rollupSeries(database, { agent_id: 'a1', granularity: 'day' }).find((r) => r.bucket_start === day && r.metric === 'runs')?.value).toBe(2); + } finally { + database.close(); + } + }); + + it('rebuildRollups restores correct rows after the table is wiped', () => { + const database = db(); + try { + ingestBatch(database, [obs(), obs({ obs_id: 'o2', status: 'error' })]); + database.exec('DELETE FROM rollups'); + const { buckets } = rebuildRollups(database); + expect(buckets).toBeGreaterThan(0); + const rows = rollupSeries(database, { agent_id: 'a1' }); + expect(rows.find((r) => r.metric === 'runs')?.value).toBe(2); + expect(rows.find((r) => r.metric === 'errors')?.value).toBe(1); + } finally { + database.close(); + } + }); +}); + +describe('traces read', () => { + it('lists traces filtered by status and agent', () => { + const database = db(); + try { + ingestBatch(database, [ + obs(), + obs({ obs_id: 'o2', trace_id: 't2', status: 'error' }), + obs({ obs_id: 'o3', trace_id: 't3', agent_id: 'a2' }), + ]); + expect(listTraces(database, { limit: 50, offset: 0 }).total).toBe(3); + expect(listTraces(database, { status: 'error', limit: 50, offset: 0 }).traces.map((t) => t.trace_id)).toEqual(['t2']); + expect(listTraces(database, { agent_id: 'a2', limit: 50, offset: 0 }).total).toBe(1); + expect(listTraces(database, { status: 'ok', limit: 50, offset: 0 }).total).toBe(2); + } finally { + database.close(); + } + }); +}); + +describe('tokens', () => { + it('creates a hashed key that verifies, and rejects wrong or missing keys', () => { + const database = db(); + try { + expect(hasTokens(database)).toBe(false); + const { key } = createToken(database, 'proj'); + expect(key.startsWith('lso_')).toBe(true); + expect(hasTokens(database)).toBe(true); + expect(verifyToken(database, key)).toBe(true); + expect(verifyToken(database, 'lso_wrong')).toBe(false); + expect(verifyToken(database, undefined)).toBe(false); + // only the hash is stored + const row = database.prepare('SELECT key_hash FROM tokens').get(); + expect(String(row?.key_hash)).not.toBe(key); + } finally { + database.close(); + } + }); +}); + +describe('transaction', () => { + it('rolls the whole batch back when a write fails', () => { + const database = db(); + try { + expect(() => + transaction(database, () => { + ingestBatchInner(database); + throw new Error('boom'); + }) + ).toThrow('boom'); + expect(observeCounts(database).observations).toBe(0); + } finally { + database.close(); + } + }); +}); + +function ingestBatchInner(database: SqlDatabase): void { + database.prepare("INSERT INTO observations (obs_id, trace_id, kind, start, updated_at) VALUES ('x','t','span',1,1)").run(); +} diff --git a/src/observe/store.ts b/src/observe/store.ts new file mode 100644 index 00000000..85f6a0e5 --- /dev/null +++ b/src/observe/store.ts @@ -0,0 +1,956 @@ +/** + * The collector's store: one SQLite file (default `~/.lousho/observe.db`) + * holding the platform's whole data plane. + * + * Why these choices: + * - WAL + `busy_timeout=5000` at open (the index contract): Studio reads + * never block the single writer, and a second reader process waits rather + * than erroring. + * - One transaction per ingest batch: the upserts and the rollup recompute + * commit or roll back together, so a crash mid-batch leaves no half-trace. + * - Upsert by row primary key everywhere (`obs_id` is the reporter's + * TraceLine.id): a resent batch is idempotent and `updated_at` gives + * last-writer-wins. + * - `rollups` is a disposable derivation, never a second write path: each + * batch recomputes the bucket rows it touches with INSERT ... SELECT ... + * GROUP BY inside the same transaction (a resend recomputes the same + * numbers, it can never double count), and `rebuildRollups()` repairs the + * whole table from `observations` on demand. + * - `usage_*`/`cost_*` are stored verbatim: the daemon never recomputes + * StepUsage, and the `cost_partial`/`usage_estimated` flags survive to + * Studio. + */ + +import { createHash, randomBytes } from 'node:crypto'; +import { mkdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { loadDatabaseSync, type SqlDatabase } from '../storage/sqlite/driver'; +import { SDKError } from '../execution/errors'; +import { migrateObserve } from './schema'; +import { OBSERVE_TRUNCATE_CHARS, type IngestEvent, type ObservationData } from './envelope'; + +const HOUR_MS = 3_600_000; +const DAY_MS = 86_400_000; + +/** Granularity labels as stored in `rollups.granularity`. */ +const ROLLUP_GRANULARITIES = { hour: HOUR_MS, day: DAY_MS } as const; +export type RollupGranularity = keyof typeof ROLLUP_GRANULARITIES; + +// The metrics `rollups` keeps per (bucket, agent) - runs, errors, cost_usd, +// input/output tokens, latency_ms - all over root observations (a run's +// `invoke_agent` span); the SELECT list in the rollup SQL is the source of truth. + +/** Open (creating the directory and file as needed), configure and migrate the observe database. */ +export function openDb(path: string): SqlDatabase { + const DatabaseSync = loadDatabaseSync(); + const location = path === ':memory:' ? path : resolve(path); + if (location !== ':memory:') mkdirSync(dirname(location), { recursive: true }); + const db = new DatabaseSync(location); + try { + if (location !== ':memory:') db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA busy_timeout = 5000'); + db.exec('PRAGMA synchronous = NORMAL'); + migrateObserve(db); + } catch (error) { + db.close(); + throw new SDKError(`Could not open the observe database at ${location}: ${(error as Error).message}`, 'LOUSHO_STORAGE_FAILED', { + cause: error, + }); + } + return db; +} + +/** `work` in one write transaction (`BEGIN IMMEDIATE`), rolled back when it throws. */ +export function transaction(db: SqlDatabase, work: () => T): T { + db.exec('BEGIN IMMEDIATE'); + try { + const result = work(); + db.exec('COMMIT'); + return result; + } catch (error) { + try { + db.exec('ROLLBACK'); + } catch { + // the transaction is already gone; keep the original error + } + throw error; + } +} + +/** Cut `value` at the contract limit; `cut` flags the row `truncated` (Studio shows it, so it must be set). */ +function truncateField(value: string | undefined, cut: { truncated: boolean }): string | null { + if (value === undefined) return null; + if (value.length <= OBSERVE_TRUNCATE_CHARS) return value; + cut.truncated = true; + return value.slice(0, OBSERVE_TRUNCATE_CHARS); +} + +const OBS_UPSERT = ` + INSERT INTO observations ( + obs_id, trace_id, session_id, agent_id, parent_obs_id, kind, name, + start, "end", status, input, output, model, + usage_input_tokens, usage_output_tokens, usage_cached_tokens, usage_reasoning_tokens, + cost_usd, cost_partial, usage_estimated, truncated, attrs, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(obs_id) DO UPDATE SET + trace_id = excluded.trace_id, + session_id = excluded.session_id, + agent_id = excluded.agent_id, + parent_obs_id = excluded.parent_obs_id, + kind = excluded.kind, + name = excluded.name, + start = excluded.start, + "end" = excluded."end", + status = excluded.status, + input = excluded.input, + output = excluded.output, + model = excluded.model, + usage_input_tokens = excluded.usage_input_tokens, + usage_output_tokens = excluded.usage_output_tokens, + usage_cached_tokens = excluded.usage_cached_tokens, + usage_reasoning_tokens = excluded.usage_reasoning_tokens, + cost_usd = excluded.cost_usd, + cost_partial = excluded.cost_partial, + usage_estimated = excluded.usage_estimated, + truncated = excluded.truncated, + attrs = excluded.attrs, + updated_at = excluded.updated_at +`; + +/** A session row is derived from whatever observations carry its id; there is no session event in the envelope. */ +const SESSION_UPSERT = ` + INSERT INTO sessions (session_id, agent_id, started_at, ended_at, status, updated_at) + VALUES (?, ?, ?, ?, 'active', ?) + ON CONFLICT(session_id) DO UPDATE SET + agent_id = COALESCE(excluded.agent_id, sessions.agent_id), + started_at = MIN(sessions.started_at, excluded.started_at), + ended_at = CASE + WHEN excluded.ended_at IS NULL THEN sessions.ended_at + WHEN sessions.ended_at IS NULL THEN excluded.ended_at + ELSE MAX(sessions.ended_at, excluded.ended_at) + END, + updated_at = excluded.updated_at +`; + +const HEARTBEAT_UPSERT = ` + INSERT INTO agents (agent_id, project, name, version, first_seen, last_heartbeat, status, meta, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'healthy', ?, ?) + ON CONFLICT(agent_id) DO UPDATE SET + project = COALESCE(excluded.project, agents.project), + name = COALESCE(excluded.name, agents.name), + version = COALESCE(excluded.version, agents.version), + last_heartbeat = MAX(COALESCE(agents.last_heartbeat, 0), excluded.last_heartbeat), + status = 'healthy', + meta = COALESCE(excluded.meta, agents.meta), + updated_at = excluded.updated_at +`; + +const DEREGISTER_UPSERT = ` + INSERT INTO agents (agent_id, first_seen, last_heartbeat, status, updated_at) + VALUES (?, ?, ?, 'stopped', ?) + ON CONFLICT(agent_id) DO UPDATE SET + status = 'stopped', + last_heartbeat = MAX(COALESCE(agents.last_heartbeat, 0), excluded.last_heartbeat), + updated_at = excluded.updated_at +`; + +const SCORE_UPSERT = ` + INSERT INTO scores (score_id, trace_id, obs_id, name, value, source, comment, agent_id, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(score_id) DO UPDATE SET + trace_id = excluded.trace_id, + obs_id = excluded.obs_id, + name = excluded.name, + value = excluded.value, + source = excluded.source, + comment = excluded.comment, + agent_id = excluded.agent_id, + updated_at = excluded.updated_at +`; + +const PROMPT_USE_UPSERT = ` + INSERT INTO prompt_uses (obs_id, prompt_id, version, created_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(obs_id, prompt_id) DO UPDATE SET + version = excluded.version +`; + +const bool = (value: boolean | undefined): number | null => (value === undefined ? null : value ? 1 : 0); + +function writeObservation(db: SqlDatabase, now: number, data: ObservationData): void { + const cut = { truncated: false }; + const input = truncateField(data.input, cut); + const output = truncateField(data.output, cut); + db.prepare(OBS_UPSERT).run( + data.obs_id, + data.trace_id, + data.session_id ?? null, + data.agent_id ?? null, + data.parent_obs_id ?? null, + data.kind, + data.name ?? null, + data.start, + data.end ?? null, + data.status ?? null, + input, + output, + data.model ?? null, + data.usage_input_tokens ?? null, + data.usage_output_tokens ?? null, + data.usage_cached_tokens ?? null, + data.usage_reasoning_tokens ?? null, + data.cost_usd ?? null, + bool(data.cost_partial), + bool(data.usage_estimated), + cut.truncated || data.truncated === true ? 1 : 0, + data.attrs === undefined ? null : JSON.stringify(data.attrs), + now + ); + if (data.session_id !== undefined) { + db.prepare(SESSION_UPSERT).run(data.session_id, data.agent_id ?? null, data.start, data.end ?? null, now); + } +} + +function writeEvent(db: SqlDatabase, now: number, event: IngestEvent): void { + switch (event.type) { + case 'agent.heartbeat': { + const d = event.data; + const ts = d.ts ?? now; + db.prepare(HEARTBEAT_UPSERT).run( + d.agent_id, + d.project ?? null, + d.name ?? null, + d.version ?? null, + ts, + ts, + JSON.stringify({ heartbeat_interval_ms: d.heartbeat_interval_ms, hostname: d.hostname, pid: d.pid, ...d.meta }), + now + ); + return; + } + case 'agent.deregister': { + const d = event.data; + const ts = d.ts ?? now; + db.prepare(DEREGISTER_UPSERT).run(d.agent_id, ts, ts, now); + return; + } + case 'observation': + writeObservation(db, now, event.data); + return; + case 'prompt_use': + db.prepare(PROMPT_USE_UPSERT).run(event.data.obs_id, event.data.prompt_id, event.data.version ?? null, now); + return; + case 'score': { + const d = event.data; + db.prepare(SCORE_UPSERT).run( + d.score_id, + d.trace_id, + d.obs_id ?? null, + d.name, + d.value ?? null, + d.source, + d.comment ?? null, + d.agent_id ?? null, + d.ts ?? now, + now + ); + return; + } + } +} + +/** + * Root-observation metrics of one bucket, per agent - recomputed wholesale + * (DELETE + INSERT ... SELECT ... GROUP BY), which is what makes a resent + * batch idempotent: it derives the same rows again rather than adding to them. + */ +function recomputeBucket(db: SqlDatabase, granularity: RollupGranularity, bucketStart: number, bucketEnd: number): void { + db.prepare('DELETE FROM rollups WHERE granularity = ? AND bucket_start = ?').run(granularity, bucketStart); + db.prepare( + `WITH roots AS ( + SELECT agent_id, status, cost_usd, usage_input_tokens, usage_output_tokens, + COALESCE("end", start) - start AS duration + FROM observations + WHERE parent_obs_id IS NULL AND agent_id IS NOT NULL AND start >= ? AND start < ? + ) + INSERT INTO rollups (bucket_start, granularity, agent_id, metric, value) + SELECT ?, ?, agent_id, 'runs', COUNT(*) FROM roots GROUP BY agent_id + UNION ALL SELECT ?, ?, agent_id, 'errors', COALESCE(SUM(status = 'error'), 0) FROM roots GROUP BY agent_id + UNION ALL SELECT ?, ?, agent_id, 'cost_usd', SUM(COALESCE(cost_usd, 0)) FROM roots GROUP BY agent_id + UNION ALL SELECT ?, ?, agent_id, 'input_tokens', SUM(COALESCE(usage_input_tokens, 0)) FROM roots GROUP BY agent_id + UNION ALL SELECT ?, ?, agent_id, 'output_tokens', SUM(COALESCE(usage_output_tokens, 0)) FROM roots GROUP BY agent_id + UNION ALL SELECT ?, ?, agent_id, 'latency_ms', SUM(duration) FROM roots GROUP BY agent_id` + ).run( + bucketStart, bucketEnd, + bucketStart, granularity, bucketStart, granularity, bucketStart, granularity, + bucketStart, granularity, bucketStart, granularity, bucketStart, granularity + ); +} + +/** Every (granularity, bucket) the batch's observations touch. */ +function affectedBuckets(batch: readonly IngestEvent[]): Array<{ granularity: RollupGranularity; bucket: number }> { + const buckets = new Map(); + for (const event of batch) { + if (event.type !== 'observation') continue; + for (const [granularity, span] of Object.entries(ROLLUP_GRANULARITIES)) { + const bucket = event.data.start - (event.data.start % span); + buckets.set(`${granularity}:${bucket}`, { granularity: granularity as RollupGranularity, bucket }); + } + } + return [...buckets.values()]; +} + +export interface IngestResult { + /** How many events the batch carried. */ + received: number; +} + +/** + * Write one validated envelope batch: all row upserts plus the rollup + * recompute of every touched bucket, in a single transaction. Idempotent on + * resend - the same batch produces the same rows, never double counts. + */ +export function ingestBatch(db: SqlDatabase, batch: readonly IngestEvent[], now: number = Date.now()): IngestResult { + return transaction(db, () => { + for (const event of batch) writeEvent(db, now, event); + for (const { granularity, bucket } of affectedBuckets(batch)) { + recomputeBucket(db, granularity, bucket, bucket + ROLLUP_GRANULARITIES[granularity]); + } + return { received: batch.length }; + }); +} + +/* --------------------------------------------------- monitoring (sweep) */ + +/** + * Heartbeat freshness thresholds, in multiples of the agent's own heartbeat + * interval (default 60s when the reporter did not declare one): `healthy` + * while the age stays within 2 intervals plus a small slack for scheduling + * jitter (60s -> ~130s), `stale` until 5 intervals (300s), `down` beyond. + * `listAgents` derives the same answer the sweep persists, so the read path + * and the stored status can never disagree. + */ +export const HEARTBEAT_INTERVAL_DEFAULT_MS = 60_000; +const FRESHNESS_STALE_FACTOR = 2; +const FRESHNESS_SLACK_MS = 10_000; +const FRESHNESS_DOWN_FACTOR = 5; + +/** `healthy`/`stale`/`down` from a heartbeat age; the sweep's and `listAgents`' shared rule. */ +export function freshnessStatus(ageMs: number, intervalMs: number): 'healthy' | 'stale' | 'down' { + if (ageMs <= FRESHNESS_STALE_FACTOR * intervalMs + FRESHNESS_SLACK_MS) return 'healthy'; + if (ageMs <= FRESHNESS_DOWN_FACTOR * intervalMs) return 'stale'; + return 'down'; +} + +/** The reporter's `heartbeat_interval_ms` out of a stored `agents.meta` JSON blob. */ +export function heartbeatIntervalFromMeta(metaJson: unknown): number | null { + if (typeof metaJson !== 'string') return null; + try { + const meta = JSON.parse(metaJson) as { heartbeat_interval_ms?: unknown }; + return typeof meta.heartbeat_interval_ms === 'number' ? meta.heartbeat_interval_ms : null; + } catch { + return null; + } +} + +/** Persist a sweep-computed status. Called only on transitions - a quiet pass writes nothing. */ +export function setAgentStatus(db: SqlDatabase, agentId: string, status: string, now: number): void { + db.prepare('UPDATE agents SET status = ?, updated_at = ? WHERE agent_id = ?').run(status, now, agentId); +} + +/** + * A crashed agent's leftover work: open sessions and still-unfinished + * observations (`end IS NULL`) flip to 'interrupted' so dead runs do not + * look alive. Called by the sweep on a transition to 'down' without a prior + * `agent.deregister`. + */ +export function interruptAgentWork(db: SqlDatabase, agentId: string, now: number): { sessions: number; observations: number } { + const sessions = db + .prepare( + "UPDATE sessions SET status = 'interrupted', ended_at = COALESCE(ended_at, ?), updated_at = ? WHERE agent_id = ? AND status = 'active'" + ) + .run(now, now, agentId); + const observations = db + .prepare('UPDATE observations SET status = \'interrupted\', updated_at = ? WHERE agent_id = ? AND "end" IS NULL') + .run(now, agentId); + return { sessions: Number(sessions.changes), observations: Number(observations.changes) }; +} + +/** + * Recompute every rollup row from `observations` (`lousho observe + * rebuild-rollups`). The table is a disposable derivation, so repair is just + * delete + the same per-bucket recompute ingest runs. + */ +export function rebuildRollups(db: SqlDatabase): { buckets: number } { + return transaction(db, () => { + db.exec('DELETE FROM rollups'); + const rows = db + .prepare('SELECT DISTINCT start FROM observations WHERE parent_obs_id IS NULL AND agent_id IS NOT NULL') + .all(); + let buckets = 0; + const done = new Set(); + for (const row of rows) { + const start = Number(row.start); + for (const [granularity, span] of Object.entries(ROLLUP_GRANULARITIES)) { + const bucket = start - (start % span); + const key = `${granularity}:${bucket}`; + if (done.has(key)) continue; + done.add(key); + recomputeBucket(db, granularity as RollupGranularity, bucket, bucket + span); + buckets++; + } + } + return { buckets }; + }); +} + +/* ------------------------------------------------------------------ tokens */ + +/** `lso_` = lousho observe; 24 random bytes is 32 base64url characters. */ +const TOKEN_PREFIX = 'lso_'; + +function hashTokenKey(key: string): string { + return createHash('sha256').update(key).digest('hex'); +} + +/** + * Create a daemon token for `project` and return the key ONCE - only its + * sha256 lands in `tokens`, so the plaintext cannot be recovered later. + */ +export function createToken(db: SqlDatabase, project: string | undefined, now: number = Date.now()): { tokenId: string; key: string } { + const tokenId = randomBytes(8).toString('hex'); + const key = `${TOKEN_PREFIX}${randomBytes(24).toString('base64url')}`; + db.prepare('INSERT INTO tokens (token_id, project, key_hash, created_at) VALUES (?, ?, ?, ?)').run( + tokenId, + project ?? null, + hashTokenKey(key), + now + ); + return { tokenId, key }; +} + +/** Whether `key` names a live token. A non-`lso_` value is cheaply rejected before the lookup. */ +export function verifyToken(db: SqlDatabase, key: string | undefined): boolean { + if (!key || !key.startsWith(TOKEN_PREFIX)) return false; + const row = db.prepare('SELECT token_id FROM tokens WHERE key_hash = ? AND revoked_at IS NULL').get(hashTokenKey(key)); + return row !== undefined; +} + +/** Whether any live token exists (used by `status` and the auth-warning). */ +export function hasTokens(db: SqlDatabase): boolean { + return db.prepare('SELECT token_id FROM tokens WHERE revoked_at IS NULL LIMIT 1').get() !== undefined; +} + +/* -------------------------------------------------------------- read path */ + +/** Row counts for `lousho observe status` (and the daemon's `/api/v1/status`). */ +export function observeCounts(db: SqlDatabase): Record { + const one = (sql: string): number => Number(db.prepare(sql).get()?.n ?? 0); + return { + agents: one('SELECT COUNT(*) AS n FROM agents'), + sessions: one('SELECT COUNT(*) AS n FROM sessions'), + observations: one('SELECT COUNT(*) AS n FROM observations'), + traces: one('SELECT COUNT(DISTINCT trace_id) AS n FROM observations'), + scores: one('SELECT COUNT(*) AS n FROM scores'), + tokens: one('SELECT COUNT(*) AS n FROM tokens WHERE revoked_at IS NULL'), + }; +} + +export interface TraceFilter { + agent_id?: string; + project?: string; + status?: string; + session_id?: string; + since?: number; + until?: number; + limit: number; + offset: number; +} + +export interface TraceRow { + trace_id: string; + name: string | null; + agent_id: string | null; + session_id: string | null; + start: number; + end: number; + status: string; + observations: number; + generations: number; + tools: number; + errors: number; + input_tokens: number; + output_tokens: number; + cost_usd: number; +} + +function traceRow(row: Record): TraceRow { + const rootStatus = row.root_status as string | null; + const errors = Number(row.error_count ?? 0); + return { + trace_id: String(row.trace_id), + name: (row.name as string | null) ?? null, + agent_id: (row.agent_id as string | null) ?? null, + session_id: (row.session_id as string | null) ?? null, + start: Number(row.start), + end: Number(row.end), + status: rootStatus ?? (errors > 0 ? 'error' : 'ok'), + observations: Number(row.observations), + generations: Number(row.generations), + tools: Number(row.tools), + errors, + input_tokens: Number(row.input_tokens), + output_tokens: Number(row.output_tokens), + cost_usd: Number(row.cost_usd), + }; +} + +/** The trace list's WHERE/HAVING pieces, shared by the page query and its total. */ +function traceFilterWhere(filter: TraceFilter): { where: string; having: string; params: (string | number | null)[] } { + const params: (string | number | null)[] = []; + let where = '1 = 1'; + const arg = (value: string | number | null): string => { + params.push(value); + return '?'; + }; + // A missing bound means "no filter"; keep the parameter count stable by binding NULL twice. + where += ` AND (${arg(filter.agent_id ?? null)} IS NULL OR o.agent_id = ${arg(filter.agent_id ?? null)})`; + where += ` AND (${arg(filter.session_id ?? null)} IS NULL OR o.session_id = ${arg(filter.session_id ?? null)})`; + where += ` AND (${arg(filter.since ?? null)} IS NULL OR o.start >= ${arg(filter.since ?? null)})`; + where += ` AND (${arg(filter.until ?? null)} IS NULL OR o.start <= ${arg(filter.until ?? null)})`; + if (filter.project !== undefined) { + where += ` AND o.agent_id IN (SELECT agent_id FROM agents WHERE project = ${arg(filter.project)})`; + } + const status = arg(filter.status ?? null); + const status2 = arg(filter.status ?? null); + // Trace status is the root observation's; a trace with no root yet is 'error' when any child failed. + // Written without select aliases so the same clause fits the bare `SELECT o.trace_id` total subquery. + const having = + `(${status} IS NULL OR COALESCE(` + + `(SELECT r.status FROM observations r WHERE r.trace_id = o.trace_id AND r.parent_obs_id IS NULL LIMIT 1), ` + + `CASE WHEN SUM(o.status = 'error') > 0 THEN 'error' ELSE 'ok' END) = ${status2})`; + return { where, having, params }; +} + +const TRACE_SELECT = ` + SELECT o.trace_id, + MIN(o.start) AS start, + MAX(COALESCE(o."end", o.start)) AS "end", + COUNT(*) AS observations, + SUM(o.kind = 'generation') AS generations, + SUM(o.kind = 'tool') AS tools, + SUM(o.status = 'error') AS error_count, + SUM(COALESCE(o.usage_input_tokens, 0)) AS input_tokens, + SUM(COALESCE(o.usage_output_tokens, 0)) AS output_tokens, + SUM(COALESCE(o.cost_usd, 0)) AS cost_usd, + MAX(o.agent_id) AS agent_id, + MAX(o.session_id) AS session_id, + (SELECT r.name FROM observations r WHERE r.trace_id = o.trace_id AND r.parent_obs_id IS NULL LIMIT 1) AS name, + (SELECT r.status FROM observations r WHERE r.trace_id = o.trace_id AND r.parent_obs_id IS NULL LIMIT 1) AS root_status + FROM observations o +`; + +export function listTraces(db: SqlDatabase, filter: TraceFilter): { traces: TraceRow[]; total: number } { + const { where, having, params } = traceFilterWhere(filter); + const grouped = `FROM observations o WHERE ${where} GROUP BY o.trace_id HAVING ${having}`; + const total = Number(db.prepare(`SELECT COUNT(*) AS n FROM (SELECT o.trace_id ${grouped})`).get(...params)?.n ?? 0); + const rows = db + .prepare(`${TRACE_SELECT.replace('FROM observations o', `FROM observations o WHERE ${where}`)} GROUP BY o.trace_id HAVING ${having} ORDER BY start DESC LIMIT ? OFFSET ?`) + .all(...params, filter.limit, filter.offset); + return { traces: rows.map(traceRow), total }; +} + +export interface ObservationRow { + obs_id: string; + trace_id: string; + session_id: string | null; + agent_id: string | null; + parent_obs_id: string | null; + kind: string; + name: string | null; + start: number; + end: number | null; + status: string | null; + input: string | null; + output: string | null; + model: string | null; + usage_input_tokens: number | null; + usage_output_tokens: number | null; + usage_cached_tokens: number | null; + usage_reasoning_tokens: number | null; + cost_usd: number | null; + cost_partial: boolean | null; + usage_estimated: boolean | null; + truncated: boolean; + attrs: Record | null; +} + +function observationRow(row: Record): ObservationRow { + let attrs: Record | null = null; + if (typeof row.attrs === 'string') { + try { + attrs = JSON.parse(row.attrs) as Record; + } catch { + attrs = null; + } + } + const flag = (value: unknown): boolean | null => (value === null || value === undefined ? null : Number(value) !== 0); + return { + obs_id: String(row.obs_id), + trace_id: String(row.trace_id), + session_id: (row.session_id as string | null) ?? null, + agent_id: (row.agent_id as string | null) ?? null, + parent_obs_id: (row.parent_obs_id as string | null) ?? null, + kind: String(row.kind), + name: (row.name as string | null) ?? null, + start: Number(row.start), + end: row.end === null || row.end === undefined ? null : Number(row.end), + status: (row.status as string | null) ?? null, + input: (row.input as string | null) ?? null, + output: (row.output as string | null) ?? null, + model: (row.model as string | null) ?? null, + usage_input_tokens: row.usage_input_tokens === null ? null : Number(row.usage_input_tokens), + usage_output_tokens: row.usage_output_tokens === null ? null : Number(row.usage_output_tokens), + usage_cached_tokens: row.usage_cached_tokens === null ? null : Number(row.usage_cached_tokens), + usage_reasoning_tokens: row.usage_reasoning_tokens === null ? null : Number(row.usage_reasoning_tokens), + cost_usd: row.cost_usd === null ? null : Number(row.cost_usd), + cost_partial: flag(row.cost_partial), + usage_estimated: flag(row.usage_estimated), + truncated: Number(row.truncated ?? 0) !== 0, + attrs, + }; +} + +export interface TraceDetail { + trace_id: string; + observations: ObservationRow[]; + scores: ScoreRow[]; +} + +/** One trace's observation tree (flat rows; the client nests by `parent_obs_id`) plus its scores. */ +export function traceDetail(db: SqlDatabase, traceId: string): TraceDetail | undefined { + const observations = db + .prepare('SELECT * FROM observations WHERE trace_id = ? ORDER BY start ASC') + .all(traceId) + .map(observationRow); + if (observations.length === 0) return undefined; + const scores = listScores(db, { trace_id: traceId, limit: 1000 }).scores; + return { trace_id: traceId, observations, scores }; +} + +export interface ScoreRow { + score_id: string; + trace_id: string; + obs_id: string | null; + name: string; + value: number | null; + source: string; + comment: string | null; + agent_id: string | null; + created_at: number; +} + +export interface ScoreFilter { + trace_id?: string; + /** Exact score name (`/`); `eval` is the prefix version. */ + name?: string; + /** Prefix match on the eval name: the score's own name or `/...`. */ + eval?: string; + agent_id?: string; + source?: string; + /** created_at bounds, milliseconds since the epoch (`since` inclusive, `until` exclusive). */ + since?: number; + until?: number; + limit: number; +} + +/** The WHERE fragment every scores read shares; `params` fills in parallel. */ +function scoreFilterWhere(filter: ScoreFilter): { where: string; params: (string | number)[] } { + let where = '1 = 1'; + const params: (string | number)[] = []; + if (filter.trace_id !== undefined) { + where += ' AND trace_id = ?'; + params.push(filter.trace_id); + } + if (filter.name !== undefined) { + where += ' AND name = ?'; + params.push(filter.name); + } + if (filter.eval !== undefined) { + // Score names are '/'; the eval's family is the exact name plus the prefix. + where += ' AND (name = ? OR name LIKE ? ESCAPE \'\\\')'; + params.push(filter.eval, `${filter.eval.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`); + } + if (filter.agent_id !== undefined) { + where += ' AND agent_id = ?'; + params.push(filter.agent_id); + } + if (filter.source !== undefined) { + where += ' AND source = ?'; + params.push(filter.source); + } + if (filter.since !== undefined) { + where += ' AND created_at >= ?'; + params.push(filter.since); + } + if (filter.until !== undefined) { + where += ' AND created_at < ?'; + params.push(filter.until); + } + return { where, params }; +} + +export function listScores(db: SqlDatabase, filter: ScoreFilter): { scores: ScoreRow[] } { + const { where, params } = scoreFilterWhere(filter); + const rows = db + .prepare(`SELECT * FROM scores WHERE ${where} ORDER BY created_at DESC LIMIT ?`) + .all(...params, filter.limit) + .map((row) => ({ + score_id: String(row.score_id), + trace_id: String(row.trace_id), + obs_id: (row.obs_id as string | null) ?? null, + name: String(row.name), + value: row.value === null ? null : Number(row.value), + source: String(row.source), + comment: (row.comment as string | null) ?? null, + agent_id: (row.agent_id as string | null) ?? null, + created_at: Number(row.created_at), + })); + return { scores: rows }; +} + +/** Pass-rate aggregate for one eval name (the score name's part before the first '/'), for the Studio eval-history view. */ +export interface ScorePassRate { + /** The eval name the group aggregates. */ + name: string; + /** Score rows in the group. */ + count: number; + /** Rows that passed: an assertion row passes when it carries no failure comment (or a perfect 1, e.g. a skipped gate). */ + passed: number; + /** `passed / count`. */ + pass_rate: number; + /** Mean `value` across the group's rows (null when every value is NULL). */ + avg_value: number | null; + /** Newest row's `created_at`. */ + last_ts: number; +} + +/** + * Pass-rate per eval name, for the eval-history view: score rows group by + * their `` prefix and each group reports its count, pass count and + * rate, mean value and last-seen time. A row counts as passed when it has no + * failure comment (assertions always write a message when they fail) or its + * value is exactly 1 (a perfect score, which covers skipped gates). + */ +export function scorePassRates( + db: SqlDatabase, + filter: { eval?: string; agent_id?: string; source?: string; since?: number; until?: number } +): ScorePassRate[] { + const { where, params } = scoreFilterWhere({ ...filter, limit: 0 }); + const rows = db + .prepare( + `SELECT + CASE WHEN instr(name, '/') > 0 THEN substr(name, 1, instr(name, '/') - 1) ELSE name END AS eval_name, + COUNT(*) AS n, + SUM(CASE WHEN comment IS NULL OR comment = '' OR value = 1 THEN 1 ELSE 0 END) AS passed, + AVG(value) AS avg_value, + MAX(created_at) AS last_ts + FROM scores WHERE ${where} + GROUP BY eval_name + ORDER BY last_ts DESC` + ) + .all(...params); + return rows.map((row) => { + const count = Number(row.n); + const passed = Number(row.passed ?? 0); + return { + name: String(row.eval_name), + count, + passed, + pass_rate: count > 0 ? passed / count : 0, + avg_value: row.avg_value === null || row.avg_value === undefined ? null : Number(row.avg_value), + last_ts: Number(row.last_ts), + }; + }); +} + +export interface AgentRow { + agent_id: string; + project: string | null; + name: string | null; + version: string | null; + first_seen: number; + last_heartbeat: number | null; + status: string; + heartbeat_age_ms: number | null; + heartbeat_interval_ms: number | null; + hostname: string | null; + pid: number | null; +} + +/** + * Fleet registry rows plus read-time freshness. The sweep (sweep.ts) rewrites + * `agents.status` on its interval; the read path derives the same value with + * the same {@link freshnessStatus} rule, so a row answers correctly between + * sweeps too: stored 'stopped' wins (a graceful exit), else the row ages + * `healthy` -> `stale` -> `down` against its heartbeat interval. + */ +export function listAgents(db: SqlDatabase, now: number = Date.now()): AgentRow[] { + const rows = db.prepare('SELECT * FROM agents ORDER BY COALESCE(last_heartbeat, first_seen) DESC').all(); + return rows.map((row) => { + let hostname: string | null = null; + let pid: number | null = null; + if (typeof row.meta === 'string') { + try { + const meta = JSON.parse(row.meta) as { hostname?: unknown; pid?: unknown }; + hostname = typeof meta.hostname === 'string' ? meta.hostname : null; + pid = typeof meta.pid === 'number' ? meta.pid : null; + } catch { + // malformed meta: freshness still derives from defaults + } + } + const interval = heartbeatIntervalFromMeta(row.meta); + const lastHeartbeat = row.last_heartbeat === null ? null : Number(row.last_heartbeat); + const age = lastHeartbeat === null ? null : Math.max(0, now - lastHeartbeat); + const step = interval ?? HEARTBEAT_INTERVAL_DEFAULT_MS; + const stored = String(row.status ?? 'healthy'); + const status = stored === 'stopped' ? 'stopped' : age === null ? stored : freshnessStatus(age, step); + return { + agent_id: String(row.agent_id), + project: (row.project as string | null) ?? null, + name: (row.name as string | null) ?? null, + version: (row.version as string | null) ?? null, + first_seen: Number(row.first_seen), + last_heartbeat: lastHeartbeat, + status, + heartbeat_age_ms: age, + heartbeat_interval_ms: interval, + hostname, + pid, + }; + }); +} + +export interface AgentInsights { + agent_id: string; + project: string | null; + name: string | null; + runs: number; + runs_per_day: number; + errors: number; + error_rate: number; + cost_usd: number; + input_tokens: number; + output_tokens: number; + p50_latency_ms: number | null; + p95_latency_ms: number | null; +} + +function percentile(sorted: number[], p: number): number | null { + if (sorted.length === 0) return null; + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); + return sorted[index]; +} + +/** + * Insights aggregates per agent over `[since, until)`: run/error/cost/token + * figures come from `rollups` (the dashboard never scans raw rows for them); + * latency percentiles are the sanctioned drilldown on `observations`. + */ +export function insightsAgents(db: SqlDatabase, windowMs: { since: number; until: number }): AgentInsights[] { + // Aggregate the day-granularity buckets inside the window; runs_per_day is + // averaged over the window's days, not only days that saw traffic. + const windowDays = Math.max(1, Math.ceil((windowMs.until - windowMs.since) / DAY_MS)); + const sinceDay = windowMs.since - (windowMs.since % DAY_MS); + const metrics = new Map>(); + for (const row of db + .prepare( + `SELECT agent_id, metric, SUM(value) AS v FROM rollups + WHERE granularity = 'day' AND bucket_start >= ? AND bucket_start < ? + GROUP BY agent_id, metric` + ) + .all(sinceDay, windowMs.until)) { + const agentId = String(row.agent_id); + const m = metrics.get(agentId) ?? new Map(); + m.set(String(row.metric), Number(row.v)); + metrics.set(agentId, m); + } + const durations = new Map(); + for (const row of db + .prepare( + `SELECT agent_id, COALESCE("end", start) - start AS d FROM observations + WHERE parent_obs_id IS NULL AND agent_id IS NOT NULL AND start >= ? AND start < ? + ORDER BY agent_id, d` + ) + .all(windowMs.since, windowMs.until)) { + const list = durations.get(String(row.agent_id)) ?? []; + list.push(Number(row.d)); + durations.set(String(row.agent_id), list); + } + const names = new Map(); + for (const agent of listAgents(db)) names.set(agent.agent_id, { project: agent.project, name: agent.name }); + const agentIds = new Set([...metrics.keys(), ...durations.keys()]); + return [...agentIds].sort().map((agentId) => { + const m = metrics.get(agentId) ?? new Map(); + const runs = m.get('runs') ?? 0; + const errors = m.get('errors') ?? 0; + const sorted = durations.get(agentId) ?? []; + return { + agent_id: agentId, + project: names.get(agentId)?.project ?? null, + name: names.get(agentId)?.name ?? null, + runs, + runs_per_day: runs / windowDays, + errors, + error_rate: runs > 0 ? errors / runs : 0, + cost_usd: m.get('cost_usd') ?? 0, + input_tokens: m.get('input_tokens') ?? 0, + output_tokens: m.get('output_tokens') ?? 0, + p50_latency_ms: percentile(sorted, 50), + p95_latency_ms: percentile(sorted, 95), + }; + }); +} + +export interface RollupRow { + bucket_start: number; + granularity: string; + agent_id: string; + metric: string; + value: number; +} + +/** Rollup time series for the Insights charts. */ +export function rollupSeries( + db: SqlDatabase, + filter: { granularity?: RollupGranularity; agent_id?: string; since?: number; until?: number } +): RollupRow[] { + let where = '1 = 1'; + const params: (string | number)[] = []; + if (filter.granularity !== undefined) { + where += ' AND granularity = ?'; + params.push(filter.granularity); + } + if (filter.agent_id !== undefined) { + where += ' AND agent_id = ?'; + params.push(filter.agent_id); + } + if (filter.since !== undefined) { + where += ' AND bucket_start >= ?'; + params.push(filter.since); + } + if (filter.until !== undefined) { + where += ' AND bucket_start < ?'; + params.push(filter.until); + } + return db + .prepare(`SELECT * FROM rollups WHERE ${where} ORDER BY bucket_start, agent_id, metric`) + .all(...params) + .map((row) => ({ + bucket_start: Number(row.bucket_start), + granularity: String(row.granularity), + agent_id: String(row.agent_id), + metric: String(row.metric), + value: Number(row.value), + })); +} diff --git a/src/observe/sweep.test.ts b/src/observe/sweep.test.ts new file mode 100644 index 00000000..0088f097 --- /dev/null +++ b/src/observe/sweep.test.ts @@ -0,0 +1,322 @@ +/** + * The monitoring child's contract: the sweep's freshness transitions + * (healthy -> stale -> down), the crash path flipping a dead agent's open + * sessions/unfinished observations to 'interrupted', webhook alerts on the + * transition and on error-rate crossings, `alerts.json` config resolution, + * and the enriched `GET /api/v1/agents` fields. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { SqlDatabase } from '../storage/sqlite/driver'; +import { loadAlertsFile, resolveAlertConfig, createAlerter, type AlertEvent } from './alerts'; +import { startObserveDaemon, type ObserveDaemon } from './daemon'; +import type { IngestEvent } from './envelope'; +import { ingestBatch, listAgents, openDb } from './store'; +import { startStatusSweep, sweepAgents } from './sweep'; + +const T0 = 1_700_000_000_000; + +const dbs: SqlDatabase[] = []; +const daemons: ObserveDaemon[] = []; +const sweeps: Array<{ stop(): void }> = []; +const tempDirs: string[] = []; + +afterEach(async () => { + while (sweeps.length > 0) sweeps.pop()?.stop(); + while (daemons.length > 0) await daemons.pop()?.close(); + while (dbs.length > 0) dbs.pop()?.close(); + while (tempDirs.length > 0) fs.rmSync(tempDirs.pop()!, { recursive: true, force: true }); +}); + +function db(): SqlDatabase { + const database = openDb(':memory:'); + dbs.push(database); + return database; +} + +const heartbeat = (agentId: string, ts: number, intervalMs = 60_000): IngestEvent => ({ + type: 'agent.heartbeat', + data: { agent_id: agentId, project: 'proj', name: 'a', ts, heartbeat_interval_ms: intervalMs }, +}); + +const obs = (over: Record = {}): IngestEvent => ({ + type: 'observation', + data: { + obs_id: 'o1', + trace_id: 't1', + session_id: 's1', + agent_id: 'a1', + kind: 'span', + name: 'invoke_agent a', + start: T0, + end: T0 + 100, + status: 'ok', + ...over, + }, +}); + +describe('sweepAgents transitions', () => { + it('ages an agent healthy -> stale -> down and persists each transition once', () => { + const database = db(); + ingestBatch(database, [heartbeat('a1', T0)], T0); + + // 60s interval: healthy within 2 intervals + 10s slack (130s) + let result = sweepAgents(database, T0 + 60_000); + expect(result.transitions).toHaveLength(0); + expect(listAgents(database, T0 + 60_000)[0].status).toBe('healthy'); + + result = sweepAgents(database, T0 + 140_000); + expect(result.transitions).toEqual([expect.objectContaining({ agent_id: 'a1', from: 'healthy', to: 'stale', crashed: false })]); + expect(result.alerts).toHaveLength(0); // stale does not alert + expect(listAgents(database, T0 + 140_000)[0].status).toBe('stale'); + + // a quiet pass rewrites nothing + expect(sweepAgents(database, T0 + 200_000).transitions).toHaveLength(0); + + result = sweepAgents(database, T0 + 310_000); + expect(result.transitions).toEqual([expect.objectContaining({ agent_id: 'a1', from: 'stale', to: 'down', crashed: true })]); + expect(listAgents(database, T0 + 310_000)[0].status).toBe('down'); + expect(result.alerts).toEqual([ + expect.objectContaining({ type: 'agent.status', agent_id: 'a1', project: 'proj', status: 'down' }), + ]); + // staying down alerts once - the transition already happened + expect(sweepAgents(database, T0 + 400_000).alerts).toHaveLength(0); + }); + + it('honors the reporter-declared heartbeat interval', () => { + const database = db(); + ingestBatch(database, [heartbeat('a1', T0, 10_000)], T0); + expect(sweepAgents(database, T0 + 26_000).transitions).toHaveLength(0); // within 2*10s+10s + expect(sweepAgents(database, T0 + 31_000).transitions[0]?.to).toBe('stale'); + expect(sweepAgents(database, T0 + 51_000).transitions[0]?.to).toBe('down'); // past 5*10s + }); + + it('never ages a deregistered (stopped) agent into down', () => { + const database = db(); + ingestBatch(database, [heartbeat('a1', T0), { type: 'agent.deregister', data: { agent_id: 'a1', ts: T0 + 5_000 } }], T0); + const result = sweepAgents(database, T0 + 9_000_000); + expect(result.transitions).toHaveLength(0); + expect(result.alerts).toHaveLength(0); + expect(listAgents(database, T0 + 9_000_000)[0].status).toBe('stopped'); + }); +}); + +describe('crash handling', () => { + it("flips a crashed agent's open sessions and unfinished observations to 'interrupted'", () => { + const database = db(); + ingestBatch( + database, + [ + heartbeat('a1', T0), + // an unfinished span (no end) - the run died mid-flight + obs({ obs_id: 'o-open', trace_id: 't-open', session_id: 's-open', end: undefined }), + // a finished one in the same agent: untouched + obs({ obs_id: 'o-done', trace_id: 't-done', session_id: 's-done' }), + ], + T0 + ); + // stale is not a crash yet + sweepAgents(database, T0 + 140_000); + expect(database.prepare('SELECT status FROM sessions WHERE session_id = ?').get('s-open')?.status).toBe('active'); + + const result = sweepAgents(database, T0 + 400_000); + expect(result.transitions[0]).toMatchObject({ to: 'down', crashed: true }); + + const session = database.prepare('SELECT status, ended_at FROM sessions WHERE session_id = ?').get('s-open'); + expect(session?.status).toBe('interrupted'); + expect(Number(session?.ended_at)).toBe(T0 + 400_000); + // every 'active' session of the dead agent flips (s-done was still 'active' too) + expect(database.prepare('SELECT status FROM sessions WHERE session_id = ?').get('s-done')?.status).toBe('interrupted'); + const open = database.prepare('SELECT status, "end" FROM observations WHERE obs_id = ?').get('o-open'); + expect(open?.status).toBe('interrupted'); + expect(open?.end).toBeNull(); // it never finished + expect(database.prepare('SELECT status FROM observations WHERE obs_id = ?').get('o-done')?.status).toBe('ok'); + + expect(result.alerts[0]).toMatchObject({ + type: 'agent.status', + agent_id: 'a1', + details: { from: 'stale', interrupted_sessions: 2, interrupted_observations: 1 }, + }); + }); +}); + +describe('error-rate alerts', () => { + it('fires once per agent per day when the rate crosses the threshold', () => { + const database = db(); + const now = Date.now(); + ingestBatch( + database, + [ + heartbeat('a1', now), + obs({ start: now, end: now + 100 }), + obs({ obs_id: 'o2', trace_id: 't2', start: now, end: now + 100, status: 'error' }), + ], + now + ); + const state = new Map(); + const opts = { errorRateThreshold: 0.4, errorRateState: state }; + const first = sweepAgents(database, now, opts); + expect(first.alerts.filter((a) => a.type === 'agent.error_rate')).toEqual([ + expect.objectContaining({ + agent_id: 'a1', + project: 'proj', + status: 'healthy', + details: expect.objectContaining({ error_rate: 0.5, threshold: 0.4, runs: 2, errors: 1 }), + }), + ]); + // same day, still over the line: no repeat + expect(sweepAgents(database, now + 1_000, opts).alerts).toHaveLength(0); + }); + + it('stays quiet under the threshold and ignores error-free days', () => { + const database = db(); + const now = Date.now(); + ingestBatch(database, [heartbeat('a1', now), obs({ start: now, end: now + 100 })], now); + const result = sweepAgents(database, now, { errorRateThreshold: 0.4, errorRateState: new Map() }); + expect(result.alerts).toHaveLength(0); + }); +}); + +describe('startStatusSweep', () => { + it('runs on the injected clock and forwards alerts to onAlerts', () => { + const database = db(); + ingestBatch(database, [heartbeat('a1', T0)], T0); + let current = T0; + const seen: AlertEvent[] = []; + const sweep = startStatusSweep(database, { intervalMs: 0, now: () => current, onAlerts: (alerts) => seen.push(...alerts) }); + sweeps.push(sweep); + expect(seen).toHaveLength(0); // boot pass: agent is fresh + current = T0 + 400_000; + const result = sweep.run(); + expect(result.transitions[0]?.to).toBe('down'); + expect(seen).toEqual([expect.objectContaining({ type: 'agent.status', agent_id: 'a1', status: 'down' })]); + }); +}); + +describe('alerts config', () => { + it('merges alerts.json webhooks with flag webhooks and validates the file', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-alerts-')); + tempDirs.push(dir); + fs.writeFileSync(path.join(dir, 'alerts.json'), JSON.stringify({ webhooks: ['http://a/hook', 'http://b/hook'], errorRateThreshold: 0.3 })); + const config = resolveAlertConfig(path.join(dir, 'observe.db'), ['http://b/hook', 'http://c/hook']); + expect(config.webhooks).toEqual(['http://a/hook', 'http://b/hook', 'http://c/hook']); + expect(config.errorRateThreshold).toBe(0.3); + }); + + it('missing alerts.json means no config; malformed JSON is a config error', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-alerts-')); + tempDirs.push(dir); + expect(resolveAlertConfig(path.join(dir, 'observe.db'))).toEqual({ webhooks: [] }); + expect(resolveAlertConfig(':memory:', ['http://x'])).toEqual({ webhooks: ['http://x'] }); + fs.writeFileSync(path.join(dir, 'alerts.json'), 'not json{'); + expect(() => loadAlertsFile(path.join(dir, 'alerts.json'))).toThrowError(/not valid JSON/); + fs.writeFileSync(path.join(dir, 'alerts.json'), JSON.stringify({ errorRateThreshold: 4 })); + expect(() => loadAlertsFile(path.join(dir, 'alerts.json'))).toThrowError(/errorRateThreshold/); + }); +}); + +describe('webhook delivery', () => { + it('POSTs the alert JSON to each webhook and warns once on failure', async () => { + const posts: { url: string; body: AlertEvent }[] = []; + const fetchFn = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + posts.push({ url: String(input), body: JSON.parse(String(init?.body)) as AlertEvent }); + return new Response('{}', { status: 200 }); + }) as unknown as typeof fetch; + const alert: AlertEvent = { + type: 'agent.status', + agent_id: 'a1', + project: 'proj', + status: 'down', + at: T0, + details: { from: 'healthy' }, + }; + createAlerter({ webhooks: ['http://hook1', 'http://hook2'] }, fetchFn).post([alert]); + expect(posts).toHaveLength(2); + expect(posts.map((p) => p.url)).toEqual(['http://hook1', 'http://hook2']); + expect(posts[0].body).toEqual(alert); + + // a dead webhook warns exactly once across passes, and never throws + const warn = vi.fn(); + const failing = vi.fn(() => Promise.reject(new Error('connect ECONNREFUSED'))) as unknown as typeof fetch; + const alerter = createAlerter({ webhooks: ['http://dead'] }, failing, warn); + alerter.post([alert]); + alerter.post([alert]); + await new Promise((resolve) => setImmediate(resolve)); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0]?.[0])).toContain('http://dead'); + }); +}); + +/* ------------------------------------------------- daemon + http surface */ + +async function start(opts: Parameters[0] = {}): Promise { + const daemon = await startObserveDaemon({ dbPath: ':memory:', port: 0, ...opts }); + daemons.push(daemon); + return daemon; +} + +const post = (daemon: ObserveDaemon, body: unknown): Promise => + fetch(`http://127.0.0.1:${daemon.port}/api/v1/ingest`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + +describe('daemon sweep + GET /api/v1/agents', () => { + it('posts the alert payload to the configured webhook when an agent goes down', async () => { + const posts: { url: string; body: AlertEvent }[] = []; + const fetchFn = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + posts.push({ url: String(input), body: JSON.parse(String(init?.body)) as AlertEvent }); + return new Response('{}', { status: 200 }); + }) as unknown as typeof fetch; + const daemon = await start({ webhooks: ['https://hooks.example/alerts'], fetchFn, sweepIntervalMs: 0 }); + expect(daemon.alerts.webhooks).toEqual(['https://hooks.example/alerts']); + + const old = Date.now() - 400_000; + await post(daemon, { + batch: [ + { type: 'agent.heartbeat', data: { agent_id: 'a1', project: 'fleet', name: 'bot', ts: old, heartbeat_interval_ms: 60_000 } }, + { type: 'observation', data: { obs_id: 'open1', trace_id: 't9', session_id: 's9', agent_id: 'a1', kind: 'span', start: old } }, + ], + }); + const result = daemon.sweep.run(); + expect(result.transitions[0]).toMatchObject({ agent_id: 'a1', from: 'healthy', to: 'down', crashed: true }); + expect(posts).toHaveLength(1); + expect(posts[0].url).toBe('https://hooks.example/alerts'); + expect(posts[0].body).toMatchObject({ + type: 'agent.status', + agent_id: 'a1', + project: 'fleet', + status: 'down', + details: { from: 'healthy', name: 'bot', interrupted_sessions: 1, interrupted_observations: 1 }, + }); + expect(daemon.db.prepare("SELECT status FROM sessions WHERE session_id = 's9'").get()?.status).toBe('interrupted'); + }); + + it('serves sweep-derived status plus runs_today / error_rate_today', async () => { + const daemon = await start({ sweepIntervalMs: 0 }); + const now = Date.now(); + await post(daemon, { + batch: [ + { type: 'agent.heartbeat', data: { agent_id: 'a1', project: 'fleet', name: 'bot', ts: now, heartbeat_interval_ms: 60_000 } }, + { type: 'agent.heartbeat', data: { agent_id: 'a2', project: 'other', ts: now - 400_000 } }, + { type: 'observation', data: { obs_id: 'o1', trace_id: 't1', agent_id: 'a1', kind: 'span', start: now, end: now + 100, status: 'ok' } }, + { type: 'observation', data: { obs_id: 'o2', trace_id: 't2', agent_id: 'a1', kind: 'span', start: now, end: now + 100, status: 'error' } }, + ], + }); + const body = (await (await fetch(`http://127.0.0.1:${daemon.port}/api/v1/agents`)).json()) as { + agents: { agent_id: string; status: string; runs_today: number; error_rate_today: number }[]; + }; + const a1 = body.agents.find((a) => a.agent_id === 'a1'); + const a2 = body.agents.find((a) => a.agent_id === 'a2'); + expect(a1).toMatchObject({ status: 'healthy', runs_today: 2, error_rate_today: 0.5 }); + expect(a2).toMatchObject({ status: 'down', runs_today: 0, error_rate_today: 0 }); + + const fleet = (await (await fetch(`http://127.0.0.1:${daemon.port}/api/v1/agents?project=fleet`)).json()) as { + agents: { agent_id: string }[]; + }; + expect(fleet.agents.map((a) => a.agent_id)).toEqual(['a1']); + }); +}); diff --git a/src/observe/sweep.ts b/src/observe/sweep.ts new file mode 100644 index 00000000..a3d0829d --- /dev/null +++ b/src/observe/sweep.ts @@ -0,0 +1,217 @@ +/** + * The monitoring child's status sweep: a daemon-side interval (default 30s) + * that rewrites `agents.status` from heartbeat freshness and emits the alerts + * the webhooks (alerts.ts) then deliver. + * + * What one pass does: + * - Recompute every non-`stopped` agent's status from `last_heartbeat` age vs + * its heartbeat interval (`freshnessStatus`, shared with the read path so + * `GET /api/v1/agents` never disagrees with the stored value). + * - Persist only transitions. `stopped` rows are skipped - a graceful + * `agent.deregister` never ages into `down`. + * - On a transition to `down` the agent crashed (no deregister landed): its + * open sessions and unfinished observations flip to `interrupted`, so dead + * runs do not look alive. + * - Check today's error rate (day rollups) against the configured threshold; + * the crossing alert fires once per agent per day bucket, so a fleet stuck + * over the line warns once, not every 30s. + */ + +import type { SqlDatabase } from '../storage/sqlite/driver'; +import type { AlertEvent } from './alerts'; +import { + HEARTBEAT_INTERVAL_DEFAULT_MS, + freshnessStatus, + heartbeatIntervalFromMeta, + interruptAgentWork, + setAgentStatus, + transaction, +} from './store'; + +/** How often the daemon sweeps; shorter than any freshness threshold. */ +const SWEEP_INTERVAL_MS = 30_000; + +const DAY_MS = 86_400_000; + +/** One agent's status rewrite in a sweep pass. `crashed` = a transition to `down`. */ +export interface AgentStatusTransition { + agent_id: string; + project: string | null; + name: string | null; + from: string; + to: 'healthy' | 'stale' | 'down'; + /** Epoch ms of the pass that made the transition. */ + at: number; + crashed: boolean; +} + +export interface SweepResult { + transitions: AgentStatusTransition[]; + /** Alerts this pass produced (status-down + error-rate crossings); the caller delivers them. */ + alerts: AlertEvent[]; +} + +export interface SweepPassOptions { + /** Today's error rate (0..1) at which an `agent.error_rate` alert fires. */ + errorRateThreshold?: number; + /** Edge memory for that alert - `agent_id` -> the day bucket already alerted on. */ + errorRateState?: Map; +} + +/** + * One sweep pass: recompute statuses, persist transitions, interrupt crashed + * agents' open work, and return the alerts this pass produced. The row + * updates run in one transaction (same rule as ingest); alert delivery is + * the caller's job so this stays synchronous and testable. + */ +export function sweepAgents(db: SqlDatabase, now: number = Date.now(), options: SweepPassOptions = {}): SweepResult { + const transitions: AgentStatusTransition[] = []; + const interrupted = new Map(); + transaction(db, () => { + for (const row of db + .prepare('SELECT agent_id, project, name, first_seen, last_heartbeat, status, meta FROM agents') + .all()) { + const stored = String(row.status ?? 'healthy'); + if (stored === 'stopped') continue; + const agentId = String(row.agent_id); + const lastHeartbeat = row.last_heartbeat === null ? null : Number(row.last_heartbeat); + // An agent that never heartbeated is fresh until first_seen says otherwise. + const anchor = lastHeartbeat ?? Number(row.first_seen); + const interval = heartbeatIntervalFromMeta(row.meta) ?? HEARTBEAT_INTERVAL_DEFAULT_MS; + const next = freshnessStatus(Math.max(0, now - anchor), interval); + if (next === stored) continue; + setAgentStatus(db, agentId, next, now); + // 'down' without a deregister is a crash: flip the agent's open work so + // dead runs stop looking alive. + if (next === 'down') interrupted.set(agentId, interruptAgentWork(db, agentId, now)); + transitions.push({ + agent_id: agentId, + project: (row.project as string | null) ?? null, + name: (row.name as string | null) ?? null, + from: stored, + to: next, + at: now, + crashed: next === 'down', + }); + } + }); + + const alerts: AlertEvent[] = []; + for (const transition of transitions) { + if (transition.to !== 'down') continue; + const flipped = interrupted.get(transition.agent_id) ?? { sessions: 0, observations: 0 }; + alerts.push({ + type: 'agent.status', + agent_id: transition.agent_id, + project: transition.project, + status: 'down', + at: transition.at, + details: { + from: transition.from, + ...(transition.name !== null ? { name: transition.name } : {}), + interrupted_sessions: flipped.sessions, + interrupted_observations: flipped.observations, + }, + }); + } + + const threshold = options.errorRateThreshold; + if (threshold !== undefined) { + const seen = options.errorRateState ?? new Map(); + // The same UTC day bucket ingest writes and `runs_today`/`error_rate_today` read. + const dayStart = now - (now % DAY_MS); + const metrics = new Map(); + for (const row of db + .prepare("SELECT agent_id, metric, value FROM rollups WHERE granularity = 'day' AND bucket_start = ?") + .all(dayStart)) { + const entry = metrics.get(String(row.agent_id)) ?? { runs: 0, errors: 0 }; + if (row.metric === 'runs') entry.runs += Number(row.value); + else if (row.metric === 'errors') entry.errors += Number(row.value); + metrics.set(String(row.agent_id), entry); + } + for (const [agentId, metric] of metrics) { + if (metric.runs === 0 || metric.errors === 0) continue; + const rate = metric.errors / metric.runs; + // Edge-triggered, once per agent per day bucket: crossing the line is + // heard once, not re-shouted every sweep while the day stays red. + if (rate < threshold || seen.get(agentId) === dayStart) continue; + seen.set(agentId, dayStart); + const agent = db.prepare('SELECT project, name, status FROM agents WHERE agent_id = ?').get(agentId); + alerts.push({ + type: 'agent.error_rate', + agent_id: agentId, + project: (agent?.project as string | null) ?? null, + status: String(agent?.status ?? 'unknown'), + at: now, + details: { + error_rate: rate, + threshold, + runs: metric.runs, + errors: metric.errors, + bucket_start: dayStart, + ...(agent?.name != null ? { name: agent.name } : {}), + }, + }); + } + } + return { transitions, alerts }; +} + +export interface StatusSweepOptions { + /** Pass cadence (default {@link SWEEP_INTERVAL_MS}); `<= 0` leaves the sweep manual-only (tests). */ + intervalMs?: number; + /** Injectable clock; default `Date.now`. */ + now?: () => number; + /** See {@link SweepPassOptions.errorRateThreshold}. */ + errorRateThreshold?: number; + /** Receives each pass's alerts; typically `alerter.post` (alerts.ts). */ + onAlerts?: (alerts: readonly AlertEvent[]) => void; +} + +export interface StatusSweep { + /** Run one pass now - the timer's tick, and the test/daemon hook for an on-demand sweep. */ + run(at?: number): SweepResult; + stop(): void; +} + +/** + * The daemon's sweep loop. The first pass runs immediately: agents that died + * while the daemon was down are marked (and reported) at boot, not one + * interval late. The timer is unref'd and swallows its own errors - a broken + * pass must neither kill nor keep alive the process. + */ +export function startStatusSweep(db: SqlDatabase, options: StatusSweepOptions = {}): StatusSweep { + const intervalMs = options.intervalMs ?? SWEEP_INTERVAL_MS; + const now = options.now ?? Date.now; + const errorRateSeen = new Map(); + const run = (at: number = now()): SweepResult => { + const result = sweepAgents(db, at, { + ...(options.errorRateThreshold !== undefined ? { errorRateThreshold: options.errorRateThreshold } : {}), + errorRateState: errorRateSeen, + }); + if (result.alerts.length > 0) options.onAlerts?.(result.alerts); + return result; + }; + try { + run(); + } catch (error) { + console.error('[lousho observe] initial sweep failed:', error); + } + const timer = + intervalMs > 0 + ? setInterval(() => { + try { + run(); + } catch (error) { + console.error('[lousho observe] sweep failed:', error); + } + }, intervalMs) + : undefined; + timer?.unref?.(); + return { + run, + stop: () => { + if (timer !== undefined) clearInterval(timer); + }, + }; +} diff --git a/src/providers/importGraph.test.ts b/src/providers/importGraph.test.ts index 7cde3a4f..2e3588ff 100644 --- a/src/providers/importGraph.test.ts +++ b/src/providers/importGraph.test.ts @@ -25,6 +25,7 @@ const NEVER_LOADED_AT_IMPORT = [ 'vue', 'svelte', // no entry imports it (the ./svelte store implements the store contract by hand) 'quickjs-emscripten', // Loaded when a codeMode run starts + 'isolated-vm', // Loaded on the first 'isolate' sandbox run (defineTool({ sandbox }) / sandbox()) '@earendil-works/pi-ai', // H2: the 'pi' provider lazy-loads it on the first pi/* call 'node:sqlite', // only the lazily-loaded /sqlite subpath may use it, and only when a store is constructed ]; diff --git a/src/sandbox/errors.ts b/src/sandbox/errors.ts new file mode 100644 index 00000000..2ee90a4b --- /dev/null +++ b/src/sandbox/errors.ts @@ -0,0 +1,41 @@ +import { SDKError } from '../utils/sdkError'; +import { brandClass } from '../utils/brand'; +import type { SandboxErrorKind } from './types'; + +/** + * A run failed at the sandbox boundary: a timeout (`kind: 'timeout'`), a + * memory-limit kill (`'oom'`) or a driver fault (`'driver'`). The `kind` + * lands on the tool span as `sandbox.error` and in the tool error result's + * `sandboxError` field. + * + * An error the sandboxed code itself threw is not a SandboxError: it crosses + * the boundary unchanged and reports as an ordinary tool error. + */ +export class SandboxError extends SDKError { + /** ToolErrorResult's `kind` for a sandboxed tool call that failed at the boundary. */ + readonly toolErrorKind = 'sandbox' as const; + + constructor( + message: string, + /** Which boundary failed (the `sandbox.error` span attribute value). */ + readonly kind: SandboxErrorKind, + /** The driver that reported it (the `sandbox.driver` span attribute value). */ + readonly driver: string, + options: { cause?: unknown } = {} + ) { + super(message, 'LOUSHO_SANDBOX_FAILED', { appendHelp: false, cause: options.cause }); + this.name = 'SandboxError'; + } +} +brandClass(SandboxError, 'SandboxError'); + +/** + * The {@link SandboxError.kind} of the first SandboxError on `error`'s + * `cause` chain, or undefined. Internal: tool-call error handling uses it to + * keep the `sandbox.error` span attribute through the error-to-result path. + */ +export function sandboxErrorKind(error: unknown, depth = 0): SandboxErrorKind | undefined { + if (depth > 4 || error === null || typeof error !== 'object') return undefined; + if (error instanceof SandboxError) return error.kind; + return sandboxErrorKind((error as { cause?: unknown }).cause, depth + 1); +} diff --git a/src/sandbox/index.ts b/src/sandbox/index.ts new file mode 100644 index 00000000..04cb7b51 --- /dev/null +++ b/src/sandbox/index.ts @@ -0,0 +1,19 @@ +/** + * The code-sandbox seam: `sandbox()` builds a box around a driver that runs + * a function's source isolated, and `defineTool({ sandbox })` routes a + * tool's `execute` through one. See docs/specs/0001-observability-platform/ + * 0001-sandboxing.md. + */ +export { sandbox } from './sandbox'; +export { SandboxError, sandboxErrorKind } from './errors'; +export { + sandboxDriverName, + type Sandbox, + type SandboxCallOptions, + type SandboxDriver, + type SandboxDriverName, + type SandboxDriverRef, + type SandboxErrorKind, + type SandboxOptions, + type ToolSandboxOptions, +} from './types'; diff --git a/src/sandbox/isolate.test.ts b/src/sandbox/isolate.test.ts new file mode 100644 index 00000000..64ecd9c7 --- /dev/null +++ b/src/sandbox/isolate.test.ts @@ -0,0 +1,140 @@ +/** + * The 'isolate' driver against an injected `isolated-vm` module, so the + * timeout/OOM/driver mapping and the memory/timeout forwarding are tested + * without the native package. + */ +import { describe, it, expect } from 'vitest'; +import { createIsolateDriver, ISOLATE_DEFAULTS } from './isolate'; +import { SandboxError } from './errors'; + +type EvalCall = { code: string; args: unknown[]; options?: { arguments?: { copy?: boolean }; result?: { copy?: boolean; promise?: boolean }; timeout?: number } }; + +interface FakeModuleOptions { + /** What evalClosure does once called (default: evaluate the caller body like a real context). */ + eval?: (isolate: FakeIsolate, code: string, args: unknown[]) => unknown; + /** Throw instead of constructing. */ + constructError?: unknown; +} + +class FakeIsolate { + static created: Array<{ memoryLimit?: number }> = []; + disposed = false; + evalCalls: EvalCall[] = []; + constructor(opts?: { memoryLimit?: number }, private behavior?: FakeModuleOptions) { + if (behavior?.constructError) throw behavior.constructError; + FakeIsolate.created.push(opts ?? {}); + } + get isDisposed(): boolean { + return this.disposed; + } + dispose(): void { + this.disposed = true; + } + async createContext() { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const isolate = this; + return { + evalCalls: this.evalCalls, + async evalClosure(code: string, args: unknown[], options?: EvalCall['options']) { + isolate.evalCalls.push({ code, args: args ?? [], options }); + if (isolate.behavior?.eval) return isolate.behavior.eval(isolate, code, args ?? []); + // What the real context does: run the caller body with $0/$1 bound, + // clone the args in, await the result, clone it back. + const [a, b] = (args ?? []).map((value) => (options?.arguments?.copy ? structuredClone(value) : value)); + const run = new Function('$0', '$1', code) as ($0: unknown, $1: unknown) => unknown; + const result = await run(a, b); + return options?.result?.copy ? structuredClone(result) : result; + }, + }; + } +} + +function fakeModule(behavior?: FakeModuleOptions): { module: never; isolates: typeof FakeIsolate.created } { + FakeIsolate.created = []; + class Bound extends FakeIsolate { + constructor(opts?: { memoryLimit?: number }) { + super(opts, behavior); + } + } + return { module: { Isolate: Bound } as never, isolates: FakeIsolate.created }; +} + +const LIMITS = { memoryMb: 64, timeoutMs: 5_000 }; + +describe('createIsolateDriver', () => { + it('creates an isolate with memoryLimit and evaluates the source with cloned args', async () => { + const { module, isolates } = fakeModule(); + const driver = createIsolateDriver(module, LIMITS); + + const result = await driver.run('(args, ctx) => ({ sum: args.a + args.b, id: ctx.toolCallId })', { + args: [{ a: 1, b: 2 }, { toolCallId: 'call_1' }], + }); + + expect(result).toEqual({ sum: 3, id: 'call_1' }); + expect(isolates).toEqual([{ memoryLimit: 64 }]); + expect(driver.name).toBe('isolate'); + }); + + it('forwards timeoutMs to evalClosure and disposes the isolate afterwards', async () => { + let last: FakeIsolate | undefined; + const { module } = fakeModule({ + eval: (isolate, code, args) => { + last = isolate; + const [a, b] = args; + const run = new Function('$0', '$1', code) as ($0: unknown, $1: unknown) => unknown; + return run(a, b); + }, + }); + const driver = createIsolateDriver(module, LIMITS); + + await driver.run('() => "done"'); + + expect(last?.evalCalls[0].options).toMatchObject({ + arguments: { copy: true }, + result: { copy: true, promise: true }, + timeout: 5000, + }); + expect(last?.disposed).toBe(true); + }); + + it('maps the wall-clock timeout to a timeout SandboxError', async () => { + const { module } = fakeModule({ eval: () => new Promise(() => undefined) }); + const driver = createIsolateDriver(module, { memoryMb: 64, timeoutMs: 25 }); + + const error = await driver.run('() => 1').catch((e: unknown) => e); + expect(error).toBeInstanceOf(SandboxError); + expect(error).toMatchObject({ kind: 'timeout', driver: 'isolate' }); + }); + + it('maps an engine-disposed isolate to an oom SandboxError', async () => { + const { module } = fakeModule({ + eval: (isolate) => { + isolate.dispose(); + throw new Error('isolate was disposed'); + }, + }); + const driver = createIsolateDriver(module, LIMITS); + + await expect(driver.run('() => 1')).rejects.toMatchObject({ kind: 'oom', driver: 'isolate' }); + }); + + it('maps a construction failure to a driver SandboxError', async () => { + const { module } = fakeModule({ constructError: new Error('no isolates for you') }); + const driver = createIsolateDriver(module, LIMITS); + + await expect(driver.run('() => 1')).rejects.toMatchObject({ kind: 'driver', driver: 'isolate' }); + }); + + it("passes the code's own error through unchanged", async () => { + const { module } = fakeModule(); + const driver = createIsolateDriver(module, LIMITS); + + await expect(driver.run('() => { throw new RangeError("the code failed") }')).rejects.toBeInstanceOf(RangeError); + }); +}); + +describe('ISOLATE_DEFAULTS', () => { + it('are the guards a sandbox without its own gets', () => { + expect(ISOLATE_DEFAULTS).toEqual({ memoryMb: 128, timeoutMs: 5_000 }); + }); +}); diff --git a/src/sandbox/isolate.ts b/src/sandbox/isolate.ts new file mode 100644 index 00000000..73e3d1da --- /dev/null +++ b/src/sandbox/isolate.ts @@ -0,0 +1,166 @@ +/** + * The `'isolate'` driver: `isolated-vm`, a V8 isolate inside the host + * process. + * + * - The package is an OPTIONAL peer, loaded on the first `run()` through + * `loadOptionalPeer()`: importing the SDK never touches it, and a missing + * install becomes a `MissingPeerDependencyError` naming `npm install + * isolated-vm`. + * - One isolate per `run()`: `Isolate({ memoryLimit })` caps its heap, the + * `evalClosure` `timeout` interrupts synchronous code and a wall-clock + * timer disposes the isolate on `timeoutMs` (the eval timeout does not + * apply while the code awaits), then the isolate is disposed either way. + * Arguments cross in and the result back by structured clone + * (`arguments: { copy }`, `result: { copy, promise }`). + * - Boundary failures map to {@link SandboxError} kinds: our own timer's + * kill is 'timeout', an isolate disposed by the engine (the memory limit) + * is 'oom' and a driver setup failure is 'driver'. An error the code + * threw crosses unchanged. + * + * `isolate` is an EXECUTION boundary, not a security boundary: the isolate + * shares the host process and an OOM can take the host down. Do not run + * code you would not run yourself; truly hostile code needs a container + * driver ('docker' / 'e2b', reserved names in {@link sandbox}). + */ + +import type * as ivm from 'isolated-vm'; +import { lazyValue, loadOptionalPeer, MissingPeerDependencyError } from '../providers/optionalPeer'; +import { SandboxError } from './errors'; +import type { SandboxCallOptions, SandboxDriver } from './types'; + +/** The resource guards of the 'isolate' driver (see `SandboxOptions`). */ +export interface IsolateLimits { + memoryMb: number; + timeoutMs: number; +} + +/** The limits a sandbox that does not set its own gets. */ +export const ISOLATE_DEFAULTS: IsolateLimits = { memoryMb: 128, timeoutMs: 5_000 }; + +/** The driver name reported as `sandbox.driver` and on SandboxError.driver. */ +const DRIVER = 'isolate'; + +/** The `isolated-vm` module surface this driver uses (the ambient declaration in typings/; the real package's types fill in when installed). */ +type IvmModule = typeof import('isolated-vm'); + +/** + * The module, loaded on first `run()`; a failed load retries next call + * (`lazyValue` does not cache a rejection). The missing-peer error names the + * driver and the alternative, not just the package. + */ +const loadIvm = lazyValue(async (): Promise => { + try { + return await loadOptionalPeer('isolated-vm', () => import('isolated-vm')); + } catch (error) { + if (error instanceof MissingPeerDependencyError) { + throw new MissingPeerDependencyError('isolated-vm', error.installCommand, { + cause: error, + feature: "the 'isolate' sandbox driver (sandbox({ driver: 'isolate' }) / defineTool({ sandbox }))", + note: 'install it, or pass another SandboxDriver instead', + }); + } + throw error; + } +}); + +/** + * The context body `run()` evaluates: the caller's `code` arrives as `$0` and + * its args as `$1`. `code` must evaluate to a function, called with those + * args; its (possibly promised) value is the run's result, cloned back. + */ +const CALLER = ` +const fn = eval('(' + $0 + ')'); +if (typeof fn !== 'function') { + throw new TypeError('sandbox run() code must evaluate to a function; the driver calls it with the given args'); +} +return fn.apply(undefined, $1); +`; + +async function runIsolated(module: IvmModule, limits: IsolateLimits, code: string, opts?: SandboxCallOptions): Promise { + let isolate: ivm.Isolate; + try { + isolate = new module.Isolate({ memoryLimit: limits.memoryMb }); + } catch (error) { + throw new SandboxError( + `the '${DRIVER}' driver could not create an isolate with memoryMb ${limits.memoryMb}`, + 'driver', + DRIVER, + { cause: error } + ); + } + let timedOut = false; + let timer: ReturnType | undefined; + // The run is bounded twice: the eval `timeout` interrupts synchronous code, + // and this timer disposes the isolate (rejecting a still-pending eval, like + // one awaiting a promise the eval timeout does not cover). + const stopped = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + timedOut = true; + isolate.dispose(); + reject(new SandboxError(`the sandboxed run exceeded timeoutMs (${limits.timeoutMs} ms)`, 'timeout', DRIVER)); + }, limits.timeoutMs); + (timer as { unref?: () => void }).unref?.(); + }); + stopped.catch(() => undefined); + try { + const context = await isolate.createContext(); + return await Promise.race([ + context.evalClosure(CALLER, [code, opts?.args ?? []], { + arguments: { copy: true }, + result: { copy: true, promise: true }, + timeout: limits.timeoutMs, + }), + stopped, + ]); + } catch (error) { + // Our timer disposed the isolate: the run outlasted timeoutMs. + if (timedOut) { + throw new SandboxError(`the sandboxed run exceeded timeoutMs (${limits.timeoutMs} ms)`, 'timeout', DRIVER, { cause: error }); + } + // The engine disposed it otherwise: the memory cap is the only configured + // limit, so a dead isolate means an OOM (or a crash we report the same way). + if (isolate.isDisposed) { + throw new SandboxError( + `the sandboxed run exceeded the isolate's memory limit (memoryMb ${limits.memoryMb}) or crashed it`, + 'oom', + DRIVER, + { cause: error } + ); + } + // The code's own error crosses unchanged (it is not a boundary failure). + throw error instanceof Error ? error : new Error(typeof error === 'string' ? error : String(error)); + } finally { + clearTimeout(timer); + if (!isolate.isDisposed) { + try { + isolate.dispose(); + } catch { + // A catastrophically failed isolate is already gone. + } + } + } +} + +/** + * The driver against an explicit `isolated-vm` module - what a test injects + * so the suite never needs the native package. `sandbox()` and + * `defineTool({ sandbox })` use {@link isolateDriver}, which lazy-loads it. + */ +export function createIsolateDriver(module: IvmModule, limits: IsolateLimits): SandboxDriver { + return { + name: DRIVER, + run: (code, opts) => runIsolated(module, limits, code, opts), + }; +} + +/** + * The `'isolate'` driver `sandbox({ driver: 'isolate' })` builds. Constructing + * it does not load `isolated-vm`; the first `run()` does + * (`MissingPeerDependencyError` when the optional peer is not installed). + */ +export function isolateDriver(limits: IsolateLimits): SandboxDriver { + return { + name: DRIVER, + run: async (code, opts) => runIsolated(await loadIvm(), limits, code, opts), + }; +} diff --git a/src/sandbox/sandbox.test.ts b/src/sandbox/sandbox.test.ts new file mode 100644 index 00000000..4227670b --- /dev/null +++ b/src/sandbox/sandbox.test.ts @@ -0,0 +1,152 @@ +/** + * sandbox(): the driver contract, structured-clone argument passing, the + * timeout race and the error kinds - with an injected driver, so no native + * `isolated-vm` is needed. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { sandbox } from './sandbox'; +import { SandboxError, sandboxErrorKind } from './errors'; +import { sandboxDriverName, type SandboxDriver } from './types'; +import { MissingPeerDependencyError } from '../providers/optionalPeer'; +import { SDKError } from '../utils/sdkError'; + +function moduleNotFound(packageName: string): Error { + return Object.assign(new Error(`Cannot find module '${packageName}'`), { code: 'MODULE_NOT_FOUND' }); +} + +afterEach(() => { + vi.doUnmock('isolated-vm'); + vi.resetModules(); +}); + +/** + * A driver that simulates the boundary with host structured clones: args are + * cloned in, the result back. `impl` lets a test observe or replace the call. + */ +function cloneDriver(impl?: (code: string, args: unknown[]) => unknown): SandboxDriver & { calls: Array<{ code: string; args: unknown[] }> } { + const calls: Array<{ code: string; args: unknown[] }> = []; + return { + name: 'fake', + calls, + async run(code, opts) { + const args = (opts?.args ?? []).map((value) => structuredClone(value)); + calls.push({ code, args }); + const result = impl ? await impl(code, args) : undefined; + return structuredClone(result); + }, + }; +} + +/** A driver that evaluates the code in-process the way 'isolate' does: `code` is a function's source, called with the args. */ +function evalDriver(): SandboxDriver { + return { + name: 'eval-fake', + async run(code, opts) { + const args = (opts?.args ?? []).map((value) => structuredClone(value)); + const fn = new Function(`return (${code})`)() as (...args: unknown[]) => unknown; + return structuredClone(await fn(...args)); + }, + }; +} + +describe('sandbox()', () => { + it('runs code through the driver and returns its result', async () => { + const box = sandbox({ driver: evalDriver() }); + expect(box.driver).toBe('eval-fake'); + await expect(box.run('(n) => n * 2', { args: [21] })).resolves.toBe(42); + }); + + it('structured-clones args in and the result back', async () => { + const seen: unknown[] = []; + const driver = cloneDriver((_code, args) => { + seen.push(...args); + (args[0] as { nested: { value: number } }).nested.value = 999; + return { out: args[0] }; + }); + const box = sandbox({ driver }); + const input = { nested: { value: 1 }, when: new Date('2026-01-01T00:00:00Z') }; + + const result = (await box.run('(x) => x', { args: [input] })) as { out: { nested: { value: number } } }; + + expect(result.out.nested.value).toBe(999); + // The caller's object was not the one mutated: the driver saw a clone. + expect(input.nested.value).toBe(1); + expect(seen[0]).not.toBe(input); + }); + + it('wraps a still-running call in a SandboxError of kind timeout after timeoutMs', async () => { + const driver: SandboxDriver = { name: 'hang', run: () => new Promise(() => undefined) }; + const box = sandbox({ driver, timeoutMs: 30 }); + + const error = await box.run('() => 1').catch((e: unknown) => e); + expect(error).toBeInstanceOf(SandboxError); + expect(error).toMatchObject({ kind: 'timeout', driver: 'hang', toolErrorKind: 'sandbox', code: 'LOUSHO_SANDBOX_FAILED' }); + expect(sandboxErrorKind(error)).toBe('timeout'); + }); + + it('lets a driver-reported oom cross unchanged', async () => { + const driver: SandboxDriver = { + name: 'oomy', + run: () => Promise.reject(new SandboxError('the isolate exceeded its memory limit', 'oom', 'oomy')), + }; + const box = sandbox({ driver }); + + await expect(box.run('() => 1')).rejects.toMatchObject({ kind: 'oom', driver: 'oomy' }); + }); + + it('lets a driver-reported fault cross with kind driver', async () => { + const driver: SandboxDriver = { + name: 'broken', + run: () => Promise.reject(new SandboxError('the sidecar refused', 'driver', 'broken')), + }; + const box = sandbox({ driver }); + + await expect(box.run('() => 1')).rejects.toMatchObject({ kind: 'driver' }); + }); + + it("passes the code's own error through as an ordinary failure", async () => { + const box = sandbox({ driver: evalDriver() }); + await expect(box.run('() => { throw new TypeError("inside") }')).rejects.toBeInstanceOf(TypeError); + }); + + it.each(['docker', 'e2b'] as const)("rejects reserved driver '%s' as not yet implemented", (driver) => { + expect(() => sandbox({ driver })).toThrowError(SDKError); + expect(() => sandbox({ driver })).toThrowError(/not yet implemented/); + }); + + it('requires a driver and positive limits', () => { + expect(() => sandbox({} as never)).toThrowError(/'driver' is required/); + expect(() => sandbox({ driver: evalDriver(), memoryMb: 0 })).toThrowError(/'memoryMb' must be a positive number/); + expect(() => sandbox({ driver: evalDriver(), timeoutMs: -1 })).toThrowError(/'timeoutMs' must be a positive number/); + expect(() => sandbox({ driver: 'nope' as never })).toThrowError(/'driver' must be 'isolate' or a SandboxDriver/); + }); + + it('defaults an unnamed driver instance to the custom name', () => { + expect(sandbox({ driver: { run: async () => 1 } }).driver).toBe('custom'); + }); + + it("lazy-loads isolated-vm on the first 'isolate' run and reports a missing peer clearly", async () => { + vi.resetModules(); + vi.doMock('isolated-vm', () => { + throw moduleNotFound('isolated-vm'); + }); + const { sandbox: freshSandbox } = await import('./sandbox'); + const box = freshSandbox({ driver: 'isolate' }); + + // Constructing the box did not touch the native module; the first run does. + const error = await box.run('() => 1').catch((e: unknown) => e); + expect(error).toBeInstanceOf(MissingPeerDependencyError); + expect(error).toMatchObject({ packageName: 'isolated-vm', installCommand: 'npm install isolated-vm' }); + expect((error as Error).message).toContain('isolate'); + expect((error as Error).message).toContain('or pass another SandboxDriver'); + }); +}); + +describe('sandboxDriverName', () => { + it('reports the configured or defaulted driver', () => { + expect(sandboxDriverName(undefined)).toBe('isolate'); + expect(sandboxDriverName('isolate')).toBe('isolate'); + expect(sandboxDriverName({ run: async () => 1 })).toBe('custom'); + expect(sandboxDriverName({ name: 'sidecar', run: async () => 1 })).toBe('sidecar'); + }); +}); diff --git a/src/sandbox/sandbox.ts b/src/sandbox/sandbox.ts new file mode 100644 index 00000000..a6d2b002 --- /dev/null +++ b/src/sandbox/sandbox.ts @@ -0,0 +1,104 @@ +/** + * `sandbox()` - the public API of the code-sandbox seam (spec + * 0001-sandboxing). A driver runs a function's source in an isolated + * context with a result channel; v1 ships `'isolate'` (isolated-vm, an + * optional peer) and reserves `'docker'`/`'e2b'`. + * + * @example + * ```ts + * const box = sandbox({ driver: 'isolate', memoryMb: 128, timeoutMs: 5_000 }); + * const result = await box.run('(input) => input * 2', { args: [21] }); + * ``` + */ + +import { SDKError } from '../utils/sdkError'; +import { SandboxError } from './errors'; +import { ISOLATE_DEFAULTS, isolateDriver } from './isolate'; +import type { Sandbox, SandboxCallOptions, SandboxDriver, SandboxDriverRef, SandboxOptions } from './types'; + +/** The resource limits a box passes to a limit-aware driver. */ +interface SandboxLimits { + memoryMb: number; + timeoutMs: number; +} + +function positive(value: unknown, name: string): void { + if (value !== undefined && !(typeof value === 'number' && Number.isFinite(value) && value > 0)) { + throw new SDKError(`sandbox(): '${name}' must be a positive number, got ${String(value)}`, 'LOUSHO_CONFIG_INVALID'); + } +} + +/** The driver for a name or instance; reserved names and anything else fail at construction. */ +function resolveDriver(ref: SandboxDriverRef, limits: SandboxLimits): SandboxDriver { + if (ref === 'isolate') return isolateDriver(limits); + if (ref === 'docker' || ref === 'e2b') { + throw new SDKError( + `sandbox(): driver '${ref}' is reserved but not yet implemented - pass a SandboxDriver instance, or use 'isolate'`, + 'LOUSHO_CONFIG_INVALID' + ); + } + if (typeof ref === 'object' && ref !== null && typeof ref.run === 'function') return ref; + throw new SDKError( + `sandbox(): 'driver' must be 'isolate' or a SandboxDriver ({ run(code, { args }) }), got ${typeof ref === 'string' ? `'${ref}'` : String(ref)}`, + 'LOUSHO_CONFIG_INVALID' + ); +} + +/** + * A wall-clock cap around `driver.run()` for one call. Drivers that can stop + * a run enforce `timeoutMs` themselves (isolate disposes the isolate); this + * race reports the same `kind: 'timeout'` for a driver that cannot, whose + * run may then still be settling in the background. + */ +function raceTimeout(name: string, limits: SandboxLimits, code: string, opts: SandboxCallOptions | undefined, run: SandboxDriver['run']): Promise { + const pending = run(code, opts); + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new SandboxError(`the sandboxed run exceeded timeoutMs (${limits.timeoutMs} ms)`, 'timeout', name)), + limits.timeoutMs + ); + pending.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + clearTimeout(timer); + reject(error); + } + ); + }); +} + +/** + * Build a sandbox: a bound driver whose `run(code, { args })` evaluates + * `code` to a function, calls it with `args` and resolves with the cloned + * result. Code, args and result cross the boundary by structured clone - + * every value must be cloneable. + * + * `{ driver: 'isolate' }` runs code in a V8 isolate (the optional + * `isolated-vm` peer - install it to use it). `'docker'` and `'e2b'` are + * reserved names: passing one throws "not yet implemented". + * + * The `'isolate'` driver is an execution boundary, NOT a security boundary: + * the isolate shares the host process and an OOM can take the host down. Do + * not run code you would not run yourself. + */ +export function sandbox(options: SandboxOptions): Sandbox { + const opts = (options ?? {}) as SandboxOptions; + positive(opts.memoryMb, 'memoryMb'); + positive(opts.timeoutMs, 'timeoutMs'); + const limits: SandboxLimits = { + memoryMb: opts.memoryMb ?? ISOLATE_DEFAULTS.memoryMb, + timeoutMs: opts.timeoutMs ?? ISOLATE_DEFAULTS.timeoutMs, + }; + if (opts.driver === undefined) { + throw new SDKError("sandbox(): 'driver' is required - 'isolate', a SandboxDriver instance, or (reserved) 'docker'/'e2b'", 'LOUSHO_CONFIG_INVALID'); + } + const driver = resolveDriver(opts.driver, limits); + const name = driver.name ?? 'custom'; + return { + driver: name, + run: (code, callOpts) => raceTimeout(name, limits, code, callOpts, (c, o) => driver.run(c, o)), + }; +} diff --git a/src/sandbox/toolSandbox.test.ts b/src/sandbox/toolSandbox.test.ts new file mode 100644 index 00000000..1da6a655 --- /dev/null +++ b/src/sandbox/toolSandbox.test.ts @@ -0,0 +1,195 @@ +/** + * defineTool({ sandbox }): the tool's execute crosses the boundary as + * source, is called with cloned `[args, ctxSubset]`, and its span carries + * `sandbox.driver` / `sandbox.error`. A fake driver evaluates the source + * in-process, so no native `isolated-vm` is needed. + */ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { defineTool } from '../tools/defineTool'; +import { ToolRegistry } from '../tools/ToolRegistry'; +import { AgentExecutor } from '../execution/AgentExecutor'; +import { AgentBuilder } from '../core'; +import { mockModel } from '../testing'; +import type { Span, TraceExporter } from '../execution/tracing'; +import { SandboxError } from './errors'; +import type { SandboxDriver } from './types'; +import type { ToolExecutionContext } from '../types'; +import { SDKError } from '../utils/sdkError'; + +/** A driver that runs `code` (the tool's serialized execute) in-process with cloned args/result. */ +function evalDriver(calls?: Array<{ code: string; args: unknown[] }>, name = 'fake'): SandboxDriver { + return { + name, + async run(code, opts) { + const args = (opts?.args ?? []).map((value) => structuredClone(value)); + calls?.push({ code, args }); + const fn = new Function(`return (${code})`)() as (...args: unknown[]) => unknown; + return structuredClone(await fn(...args)); + }, + }; +} + +const input = z.object({ x: z.number() }); + +function ctx(partial: Partial = {}): ToolExecutionContext { + return { toolCallId: 'call_1', messages: [], ...partial } as ToolExecutionContext; +} + +describe('defineTool({ sandbox })', () => { + it('runs execute inside the box with cloned args and the cloneable ctx subset', async () => { + const calls: Array<{ code: string; args: unknown[] }> = []; + const tool = defineTool({ + name: 'doubler', + description: 'Double x', + input, + sandbox: { driver: evalDriver(calls) }, + execute: async (args, c) => ({ doubled: args.x * 2, callId: (c as { toolCallId?: string }).toolCallId }), + }); + + const arg = { x: 21 }; + const result = await tool.execute(arg, ctx({ sessionId: 's1' })); + + expect(result).toEqual({ doubled: 42, callId: 'call_1' }); + // The tool's own source crossed the boundary, not a wrapper. + expect(calls[0].code).toContain('doubled'); + // The ctx subset carries toolCallId/sessionId and NOT the live members. + const innerCtx = calls[0].args[1] as Record; + expect(innerCtx.toolCallId).toBe('call_1'); + expect(innerCtx.sessionId).toBe('s1'); + expect('abortSignal' in innerCtx).toBe(false); + expect('getToken' in innerCtx).toBe(false); + // The sandbox field on the descriptor marks it for the span attributes. + expect(tool.sandbox).toEqual({ driver: expect.objectContaining({ name: 'fake' }) }); + }); + + it('rejects with the SandboxError a boundary failure raised', async () => { + const tool = defineTool({ + name: 'too_slow', + description: 'Never returns', + input, + sandbox: { driver: { name: 'hang', run: () => new Promise(() => undefined) }, timeoutMs: 30 }, + execute: async () => ({ ok: true }), + }); + + await expect(tool.execute({ x: 1 }, ctx())).rejects.toMatchObject({ kind: 'timeout', toolErrorKind: 'sandbox' }); + }); + + it("fails a sandboxed tool's own error as an ordinary tool error", async () => { + const tool = defineTool({ + name: 'failing', + description: 'Throws', + input, + sandbox: { driver: evalDriver() }, + execute: async () => { + throw new RangeError('the tool failed'); + }, + }); + + await expect(tool.execute({ x: 1 }, ctx())).rejects.toBeInstanceOf(RangeError); + }); + + it('rejects a reserved driver name at definition time', () => { + expect(() => + defineTool({ name: 'boxed', description: 'x', input, sandbox: { driver: 'docker' }, execute: async () => ({}) }) + ).toThrowError(/not yet implemented/); + }); + + it('rejects sandbox combined with requiresSandbox/sandboxExecute or a generator execute', () => { + expect(() => + defineTool({ + name: 'both', + description: 'x', + input, + requiresSandbox: true, + sandboxExecute: async () => ({}), + sandbox: { driver: evalDriver() }, + execute: async () => ({}), + }) + ).toThrowError(SDKError); + expect(() => + defineTool({ + name: 'streamy', + description: 'x', + input, + sandbox: { driver: evalDriver() }, + execute: async function* () { + yield {}; + }, + }) + ).toThrowError(/generator/); + }); + + it('rejects a bound execute at definition time', () => { + const bound = (async () => ({})).bind(null); + expect(() => defineTool({ name: 'bound', description: 'x', input, sandbox: { driver: evalDriver() }, execute: bound })).toThrowError( + /cannot be sandboxed/ + ); + }); +}); + +/** In-memory exporter: remembers every span, in start order. */ +function memoryExporter() { + const spans: Span[] = []; + const exporter: TraceExporter = { + onSpanStart: (span) => spans.push(span), + onSpanEnd: () => undefined, + }; + return { exporter, spans }; +} + +async function runAgentWithTool(tool: ReturnType) { + const toolRegistry = new ToolRegistry(); + toolRegistry.register(tool); + const agent = { + ...AgentBuilder.create().setName('Box Bot').addTool(tool.name, { tool: tool.name, options: {} }).build(), + id: 'agent-1', + prompt: 'You are terse.', + settings: { model: 'test-model' }, + }; + const { exporter, spans } = memoryExporter(); + const provider = mockModel([ + { toolCalls: [{ id: 'call_1', name: tool.name, args: { x: 3 } }], usage: { inputTokens: 1, outputTokens: 1 } }, + { text: 'done', usage: { inputTokens: 1, outputTokens: 1 } }, + ]); + await AgentExecutor.execute({ agent, input: 'go', provider, toolRegistry, exporter }); + return spans; +} + +describe('sandboxed tool spans', () => { + it('records sandbox.driver on the execute_tool span', async () => { + const tool = defineTool({ + name: 'boxed_double', + description: 'Double x', + input, + sandbox: { driver: evalDriver() }, + execute: async (args) => ({ doubled: (args as { x: number }).x * 2 }), + }); + + const spans = await runAgentWithTool(tool); + + const toolSpan = spans.find((span) => span.name === 'execute_tool boxed_double'); + expect(toolSpan?.attributes['sandbox.driver']).toBe('fake'); + expect(toolSpan?.attributes['sandbox.error']).toBeUndefined(); + expect(toolSpan?.status).toBeUndefined(); + }); + + it('records sandbox.error when the boundary fails', async () => { + const tool = defineTool({ + name: 'boxed_hang', + description: 'Hangs', + input, + sandbox: { + driver: { name: 'hang', run: () => Promise.reject(new SandboxError('out of memory', 'oom', 'hang')) }, + }, + execute: async () => ({ ok: true }), + }); + + const spans = await runAgentWithTool(tool); + + const toolSpan = spans.find((span) => span.name === 'execute_tool boxed_hang'); + expect(toolSpan?.attributes['sandbox.driver']).toBe('hang'); + expect(toolSpan?.attributes['sandbox.error']).toBe('oom'); + expect(toolSpan?.status?.code).toBe('error'); + }); +}); diff --git a/src/sandbox/toolSandbox.ts b/src/sandbox/toolSandbox.ts new file mode 100644 index 00000000..3c479708 --- /dev/null +++ b/src/sandbox/toolSandbox.ts @@ -0,0 +1,82 @@ +/** + * The bridge between `defineTool({ sandbox })` and {@link sandbox}: a tool's + * `execute` crosses the boundary as source text (a live closure cannot), + * its args and the cloneable part of its `ctx` as structured clones, and + * its result clones back. + * + * What the sandboxed `execute` sees: + * + * - `args`: the validated arguments, cloned. + * - `ctx`: the cloneable subset `{ toolCallId, messages, sessionId, + * principal, approval }`. The non-cloneable members of + * `ToolExecutionContext` cannot cross: `abortSignal`, `getToken`, + * `requireAuth` and `onDelegatedUsage` are simply absent inside the box, + * so a tool that needs OAuth tokens or cancellation cannot be sandboxed + * this way. + * - Globals: whatever the driver provides. The 'isolate' driver is a bare + * V8 context - ECMAScript builtins only, no `fetch`, `console`, `process` + * or host modules - so `execute` must be self-contained: every function or + * value it references must be declared inside its own source. A closure + * over host state fails inside the box as an ordinary tool error. + * + * What comes back is `execute`'s resolved value, cloned; what it threw + * crosses back as a normal tool error (the model sees it). A boundary + * failure crosses as a {@link SandboxError}. + */ + +import { SDKError } from '../utils/sdkError'; +import type { ToolExecutionContext } from '../types/tool'; +import type { Sandbox } from './types'; + +/** `execute` as source text for the driver, or a defineTool-time failure when it cannot cross the boundary. */ +export function toolSandboxSource(toolName: string, execute: unknown): string { + if (typeof execute !== 'function') { + throw new SDKError(`defineTool: tool '${toolName}' with 'sandbox' needs an 'execute' function`, 'LOUSHO_CONFIG_INVALID'); + } + const ctor = (execute as { constructor?: { name?: unknown } }).constructor?.name; + const source = Function.prototype.toString.call(execute).trim(); + if ( + ctor === 'GeneratorFunction' || + ctor === 'AsyncGeneratorFunction' || + /^(?:async\s+)?(?:function\s*)?\*/.test(source) + ) { + throw new SDKError( + `defineTool: tool '${toolName}' cannot combine 'sandbox' with a generator 'execute' - a sandboxed run returns one value, not a stream`, + 'LOUSHO_CONFIG_INVALID' + ); + } + if (/{\s*\[native code\]\s*}/.test(source)) { + throw new SDKError( + `defineTool: tool '${toolName}' has an 'execute' that cannot be sandboxed (native or bound function - its source cannot cross the boundary)`, + 'LOUSHO_CONFIG_INVALID' + ); + } + // Object-method shorthand (`async execute(args) {`) does not evaluate as an + // expression the way `function`/`async`/arrow forms do - give it the keyword. + // The arrow forms `async (args) =>` and `x =>` are left alone (a method + // literally named `async` is not supported). + if (/^async\s+[A-Za-z_$][\w$]*\s*\(/.test(source)) return source.replace(/^async\s+/, 'async function '); + if (/^(?!async\b)[A-Za-z_$][\w$]*\s*\(/.test(source)) return `function ${source}`; + return source; +} + +/** + * The part of `ctx` that crosses the boundary by structured clone. A copy - + * mutations inside the box do not reach the host object, and the live + * members (`abortSignal`, `getToken`, `requireAuth`, `onDelegatedUsage`) are + * left out. + */ +function sandboxableContext(ctx: ToolExecutionContext): Record { + return { + toolCallId: ctx.toolCallId, + messages: ctx.messages, + ...(ctx.sessionId !== undefined && { sessionId: ctx.sessionId }), + ...(ctx.principal !== undefined && { principal: ctx.principal }), + ...(ctx.approval !== undefined && { approval: ctx.approval }), + }; +} + +/** One sandboxed tool call: `execute` (as `source`) runs inside `box` with `[args, ctxSubset]`. */ +export function runToolInSandbox(box: Sandbox, source: string, args: unknown, ctx: ToolExecutionContext): Promise { + return box.run(source, { args: [args, sandboxableContext(ctx)] }); +} diff --git a/src/sandbox/types.ts b/src/sandbox/types.ts new file mode 100644 index 00000000..8b444440 --- /dev/null +++ b/src/sandbox/types.ts @@ -0,0 +1,114 @@ +/** + * The types of the code-sandbox seam. A {@link SandboxDriver} runs a piece of + * code in an isolated context and resolves with its result; {@link sandbox} + * and `defineTool({ sandbox })` build on it. + * + * This module is types plus one pure helper and imports nothing, so any part + * of the SDK (tool descriptors, span attributes) can reference it without + * pulling a driver implementation. + */ + +/** + * Why a sandboxed run failed at the boundary - the `sandbox.error` span + * attribute values: 'timeout' (the run outlasted `timeoutMs`), 'oom' (the + * driver killed it for its memory cap) or 'driver' (the driver itself + * failed). An error the sandboxed code threw is not a boundary failure and + * carries none of these. + */ +export type SandboxErrorKind = 'timeout' | 'oom' | 'driver'; + +/** Per-call options of {@link SandboxDriver.run} and {@link Sandbox.run}. */ +export interface SandboxCallOptions { + /** + * The arguments the code's function is called with, in order. Every value + * crosses the boundary by structured clone, so each must be cloneable: + * plain JSON values, ArrayBuffer and typed arrays, Date, Map, Set, ... - + * never functions, class instances, AbortSignal or other live references. + */ + args?: unknown[]; +} + +/** + * A driver runs `code` in its isolated context. `code` evaluates to a + * function (an arrow, `function` or `async function` expression), which the + * driver calls with `opts.args` (`[]` when absent); `run()` resolves with + * what it returns - a promise it returns is awaited - cloned back across the + * boundary, so the value must be structured-cloneable. + * + * A driver reports a boundary failure - a timeout, a memory-limit kill or + * its own fault - as a {@link SandboxError} whose `kind` names it; an error + * the code itself throws crosses as itself (it is a tool error, not a + * boundary failure). + */ +export interface SandboxDriver { + /** + * The driver's name, recorded on tool spans as `sandbox.driver`. A driver + * instance without one reports `'custom'`. + */ + readonly name?: string; + /** + * Evaluate `code` to a function, call it with `opts.args` and resolve with + * its result (structured-cloned back). Boundary failures reject with a + * `SandboxError` whose `kind` names the cause. + */ + run(code: string, opts?: SandboxCallOptions): Promise; +} + +/** The drivers {@link sandbox} knows by name. `'docker'` and `'e2b'` are reserved: naming one errors "not yet implemented". */ +export type SandboxDriverName = 'isolate' | 'docker' | 'e2b'; + +/** What `sandbox({ driver })` and `defineTool({ sandbox: { driver } })` take: a {@link SandboxDriverName} or a driver instance. */ +export type SandboxDriverRef = SandboxDriverName | SandboxDriver; + +/** Options of {@link sandbox}. */ +export interface SandboxOptions { + /** + * Which driver runs the code: `'isolate'` (the default elsewhere - V8 + * isolates via the optional `isolated-vm` peer), a {@link SandboxDriver} + * instance, or a reserved name (`'docker'`, `'e2b'`) which errors + * "not yet implemented". + */ + driver: SandboxDriverRef; + /** + * Heap cap for one run, in MB, enforced by the driver when it supports one + * (`'isolate'` does). A breach fails the run with `kind: 'oom'` - and can + * take the host process: `isolate` is an execution boundary, not a + * security one. + */ + memoryMb?: number; + /** + * Wall-clock cap of one run, in ms; a run still going then fails with + * `kind: 'timeout'`. Drivers that can stop the run do so (`isolate` + * disposes the isolate); for a custom driver the box reports the timeout + * while the run itself may still be settling in the background. + */ + timeoutMs?: number; +} + +/** What {@link sandbox} returns: a bound driver ready for `run()` calls. */ +export interface Sandbox { + /** The resolved driver name: `'isolate'`, or the driver instance's `name`. */ + readonly driver: string; + /** Same contract as {@link SandboxDriver.run}. */ + run(code: string, opts?: SandboxCallOptions): Promise; +} + +/** + * `defineTool({ sandbox })`: run the tool's `execute` inside a code sandbox. + * The contract (what crosses the boundary, which `ctx` fields survive) is on + * {@link import('../types').ToolDescriptor.sandbox}. + */ +export interface ToolSandboxOptions { + /** Default `'isolate'`; `'docker'` and `'e2b'` are reserved and error "not yet implemented". */ + driver?: SandboxDriverRef; + /** The driver's heap cap for one call, in MB (see {@link SandboxOptions.memoryMb}). */ + memoryMb?: number; + /** Wall-clock cap of one call, in ms (see {@link SandboxOptions.timeoutMs}). */ + timeoutMs?: number; +} + +/** The `sandbox.driver` attribute value for a sandbox option's driver (`'isolate'` when unset). */ +export function sandboxDriverName(driver: SandboxDriverRef | undefined): string { + if (driver === undefined) return 'isolate'; + return typeof driver === 'string' ? driver : driver.name ?? 'custom'; +} diff --git a/src/server/chatRoutes.ts b/src/server/chatRoutes.ts index 2d2d9fd9..bce6a596 100644 --- a/src/server/chatRoutes.ts +++ b/src/server/chatRoutes.ts @@ -8,19 +8,16 @@ import type * as http from 'node:http'; import { Readable } from 'node:stream'; import { handleChatFetch, PayloadTooLargeError, type ChatRoutesContext } from './fetchRoutes'; +import { sendJson } from './sendJson'; export type { ChatRoutesContext } from './fetchRoutes'; +export { sendJson }; export function sendText(res: http.ServerResponse, status: number, text: string): void { res.writeHead(status, { 'Content-Type': 'text/plain' }); res.end(text); } -export function sendJson(res: http.ServerResponse, status: number, value: unknown): void { - res.writeHead(status, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(value)); -} - /** `req.headers` (Node `IncomingMessage`) as Fetch `Headers` — multi-values flattened. */ export function incomingHeaders(req: http.IncomingMessage): Headers { const headers = new Headers(); diff --git a/src/server/sendJson.ts b/src/server/sendJson.ts new file mode 100644 index 00000000..9ecdab63 --- /dev/null +++ b/src/server/sendJson.ts @@ -0,0 +1,7 @@ +import type * as http from 'node:http'; + +/** Write `value` as a JSON response body with the given status. */ +export function sendJson(res: http.ServerResponse, status: number, value: unknown): void { + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(value)); +} diff --git a/src/tools/defineTool.ts b/src/tools/defineTool.ts index a28d409c..fdff2f54 100644 --- a/src/tools/defineTool.ts +++ b/src/tools/defineTool.ts @@ -5,6 +5,9 @@ import { isModelSchema, nonObjectRoot, unrepresentableFields, type InferSchemaOu import { sharedWeakSet } from '../utils/brand'; import { SDKError, ToolExecutionError } from '../execution/errors'; import { unknownOptionKey } from '../utils/knownOptionKeys'; +import { sandbox } from '../sandbox/sandbox'; +import type { Sandbox, ToolSandboxOptions } from '../sandbox/types'; +import { runToolInSandbox, toolSandboxSource } from '../sandbox/toolSandbox'; /** Tool names must satisfy the constraint LLM providers impose on function names. */ const TOOL_NAME_PATTERN = /^[a-zA-Z0-9_-]{1,64}$/; @@ -65,6 +68,21 @@ export interface DefineToolOptions { requiresSandbox?: boolean; /** Sandboxed execution path used instead of `execute` when `requiresSandbox` is true. */ sandboxExecute?: (args: InferSchemaOutput, sandbox: SandboxAdapter) => Promise; + /** + * Run `execute` inside a code sandbox instead of in-process (spec + * 0001-sandboxing). `{ driver: 'isolate' }` (the default) uses the + * optional `isolated-vm` peer; `{ driver }` may also be a `SandboxDriver` + * instance (`'docker'`/`'e2b'` are reserved). `execute`'s source crosses + * the boundary and is called there with `(args, ctxSubset)` - everything + * crossing is structured-cloned, so `execute` must be self-contained (no + * host closures), and `ctx` inside the box only has the cloneable fields + * (`toolCallId`, `messages`, `sessionId`, `principal`, `approval`; no + * `abortSignal`/`getToken`/`requireAuth`). `memoryMb`/`timeoutMs` bound + * each call; a breach fails the call with a `kind: 'sandbox'` tool error. + * Cannot be combined with `requiresSandbox`/`sandboxExecute` or a + * streaming (generator) `execute`. + */ + sandbox?: ToolSandboxOptions; /** * Runs the tool. Arguments are typed from `input`; the return type is preserved on the result. * May be an `async function*`: each `yield` is a complete snapshot of the output, streamed @@ -128,6 +146,7 @@ const DEFINE_TOOL_KEYS = { timeoutMs: true, requiresSandbox: true, sandboxExecute: true, + sandbox: true, execute: true, } as const satisfies Record, true>; @@ -202,6 +221,29 @@ function assertValidOptions(opts: { name?: unknown; description?: unknown; input } } +/** + * The sandbox a `{ sandbox }` tool runs in, built now so a bad driver name or + * limit fails at definition time; `isolated-vm` itself still loads on first + * call. Also returns `execute`'s source text (what crosses the boundary). + */ +function sandboxedExecute( + opts: { name: string; sandbox?: ToolSandboxOptions; requiresSandbox?: boolean; sandboxExecute?: unknown; execute: unknown } +): { box: Sandbox; source: string } | undefined { + if (opts.sandbox === undefined) return undefined; + if (typeof opts.sandbox !== 'object' || opts.sandbox === null) { + fail(`tool '${opts.name}': 'sandbox' must be an object like { driver: 'isolate' }`, `Example: sandbox: { driver: 'isolate', memoryMb: 128, timeoutMs: 5_000 }`); + } + if (opts.requiresSandbox || opts.sandboxExecute !== undefined) { + fail( + `tool '${opts.name}': 'sandbox' cannot be combined with 'requiresSandbox'/'sandboxExecute' - those route to a SandboxAdapter, this one runs the execute source in a code sandbox`, + `Keep 'sandbox' (code isolation), or drop it and use 'requiresSandbox' + 'sandboxExecute' (a SandboxAdapter path the tool drives itself)` + ); + } + const source = toolSandboxSource(opts.name, opts.execute); + const box = sandbox({ driver: opts.sandbox.driver ?? 'isolate', memoryMb: opts.sandbox.memoryMb, timeoutMs: opts.sandbox.timeoutMs }); + return { box, source }; +} + /** * `run` with `ctx.abortSignal` also aborted after `timeoutMs`: the call * rejects then (with a `ToolExecutionError` the model sees), or as soon as the @@ -298,10 +340,18 @@ export function defineTool( ): DefinedTool, Awaited> { assertValidOptions(opts); + // `sandbox` sends `execute`'s source into the box on each call; a reserved + // driver name or a non-serializable `execute` fails here, at definition time. + const sandboxed = sandboxedExecute(opts); // An async generator passes through unchanged (it is not thenable); the runtime iterates it. const { timeoutMs } = opts; - const execute = async (args: InferSchemaOutput, ctx: ToolExecutionContext): Promise> => - timeoutMs === undefined ? await opts.execute(args, ctx) : await withTimeout(opts.name, timeoutMs, ctx, (timed) => opts.execute(args, timed)); + const execute = async (args: InferSchemaOutput, ctx: ToolExecutionContext): Promise> => { + const call = (c: ToolExecutionContext): Promise => + sandboxed === undefined ? Promise.resolve(opts.execute(args, c)) : runToolInSandbox(sandboxed.box, sandboxed.source, args, c); + const result: unknown = + timeoutMs === undefined ? await call(ctx) : await withTimeout(opts.name, timeoutMs, ctx, call); + return result as Awaited; + }; // legacy (.tool): the `ai` v4 Tool shape. Removed in D26. const legacyTool = legacyAiTool(opts.description, opts.input, execute as NonNullable); const defined: DefinedTool, Awaited> = { @@ -315,6 +365,7 @@ export function defineTool( needsApproval: opts.needsApproval, requiresSandbox: opts.requiresSandbox, sandboxExecute: opts.sandboxExecute as ToolDescriptor['sandboxExecute'], + ...(opts.sandbox !== undefined && { sandbox: opts.sandbox }), ...(opts.deferLoading !== undefined && { deferLoading: opts.deferLoading }), ...toolMetadata(opts), }; diff --git a/src/traces/index.ts b/src/traces/index.ts index 9f45c379..856f9582 100644 --- a/src/traces/index.ts +++ b/src/traces/index.ts @@ -6,3 +6,13 @@ export { fileTraceExporter, type FileTraceExporterOptions } from './fileTraceExp export { listTraces, readTrace, type ListTracesOptions, type TraceSummary } from './readTraces'; export { DEFAULT_TRACE_DIR, type TraceLine } from './format'; export type { Span, SpanKind, SpanStatus, TraceExporter } from '../execution/tracing'; +// The collector reporter (LOUSHO_OBSERVE): node builtins are lazy inside, so +// this export stays Worker-bundle-safe while letting runs wire it explicitly. +export { + observeExporter, + observeExporterFromEnv, + fanOutExporters, + withObserveExporter, + type ObserveExporterOptions, + type ObserveReporter, +} from '../observe/reporter'; diff --git a/src/types/tool.ts b/src/types/tool.ts index eb84b27d..4d975695 100644 --- a/src/types/tool.ts +++ b/src/types/tool.ts @@ -3,6 +3,7 @@ import type { StandardSchemaV1 } from '../utils/zodCompat'; import type { RunUsage } from '../models/usage'; import type { Message } from '../providers/llm'; import { SandboxAdapter } from '../security/sandboxCore'; +import type { ToolSandboxOptions } from '../sandbox/types'; import type { Principal } from '../auth/types'; import type { OAuthProvider } from '../oauth/defineOAuthProvider'; import type { OAuthToken } from '../oauth/types'; @@ -242,6 +243,16 @@ export interface ToolDescriptor { sandbox: SandboxAdapter, ctx?: ToolExecutionContext ) => Promise; + /** + * The code sandbox `execute` runs in - set by `defineTool({ sandbox })` + * (spec 0001-sandboxing). The `execute_tool` span records the driver as + * `sandbox.driver`, and a boundary failure as `sandbox.error` + * (`'timeout' | 'oom' | 'driver'`). The field is how the tool declared it + * runs isolated; the actual routing happens inside the `execute` wrapper + * `defineTool` built (the driver sees the function's source, not the live + * closure), so setting it on a hand-built descriptor only labels it. + */ + sandbox?: ToolSandboxOptions; } /** diff --git a/src/utils/errorCodes.ts b/src/utils/errorCodes.ts index 3b8b44c8..85f9dcc3 100644 --- a/src/utils/errorCodes.ts +++ b/src/utils/errorCodes.ts @@ -108,6 +108,8 @@ export const ERROR_CODES = { LOUSHO_SANDBOX_PATH_DENIED: "Write under the sandbox's root (SubprocessSandbox({ root })), or pass the root that should contain the path.", LOUSHO_SANDBOX_EGRESS_UNSUPPORTED: "Run on Docker Engine 25.0.5+ for Linux on this host (not Docker Desktop, rootless or a remote daemon), or use network: 'none'.", + LOUSHO_SANDBOX_FAILED: + "The error's `kind` says which boundary failed ('timeout', 'oom' or 'driver'); raise the limit in sandbox()/defineTool({ sandbox }), or check the driver.", LOUSHO_AGENT_EXECUTION_FAILED: 'Look at the `cause` for the underlying failure.', LOUSHO_FLOW_EXECUTION_FAILED: 'Look at the failing `step` and the `cause`.', LOUSHO_VALIDATION_FAILED: 'Fix the fields listed in `errors`.', diff --git a/tsup.config.ts b/tsup.config.ts index 4bbd169a..ff876eae 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -40,6 +40,8 @@ const entry = { 'cli/eval': 'src/cli/eval.ts', 'cli/init': 'src/cli/init.ts', 'cli/traces': 'src/cli/traces.ts', + 'cli/observe': 'src/cli/observe.ts', + 'cli/prompt': 'src/cli/prompt.ts', 'execution/otel': 'src/execution/otel.ts', 'execution/hooks': 'src/execution/hooks.ts', 'storage/sqlite/index': 'src/storage/sqlite/index.ts', @@ -81,7 +83,7 @@ export default defineConfig([ treeshake: true, minify: false, // Optional peers stay external too: tsup would externalize them from package.json anyway. - external: ['ai', 'zod', '@opentelemetry/api', 'dockerode', '@modelcontextprotocol/sdk', 'prompts', 'quickjs-emscripten', '@earendil-works/pi-ai'], + external: ['ai', 'zod', '@opentelemetry/api', 'dockerode', '@modelcontextprotocol/sdk', 'prompts', 'quickjs-emscripten', 'isolated-vm', '@earendil-works/pi-ai'], onSuccess: async () => { copyDevUi(); }, diff --git a/typings/optional-peers.d.ts b/typings/optional-peers.d.ts index f3cf9c19..72176c47 100644 --- a/typings/optional-peers.d.ts +++ b/typings/optional-peers.d.ts @@ -34,3 +34,28 @@ declare module 'ollama-ai-provider-v2' { // eslint-disable-next-line @typescript-eslint/no-explicit-any export function createOllama(options?: Record): any; } + +// The 'isolate' sandbox driver's optional peer (src/sandbox/isolate.ts) - a +// native module, so it is an optional peer but NOT a devDependency; the +// driver lazy-loads it and tests inject a fake module. Only the surface the +// driver uses is declared; the real package's types fill in when installed. +declare module 'isolated-vm' { + /** Options of Context.evalClosure (the parts src/sandbox/isolate.ts passes). */ + export interface EvalClosureOptions { + /** Structured-clone the arguments into the isolate. */ + arguments?: { copy?: boolean }; + /** Structured-clone the result back; `promise` awaits a returned promise inside the isolate. */ + result?: { copy?: boolean; promise?: boolean }; + /** Interrupt synchronous execution after this many ms. */ + timeout?: number; + } + export class Context { + evalClosure(code: string, args?: unknown[], options?: EvalClosureOptions): Promise; + } + export class Isolate { + constructor(options?: { memoryLimit?: number }); + createContext(): Promise; + dispose(): void; + readonly isDisposed: boolean; + } +}