diff --git a/README.md b/README.md index 8bac76f09..a1b25f4b6 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,7 @@ console.log(result.usage.totalTokens, result.finishReason, result.steps); - [Configuration](docs/configuration.md) - agent spec fields, provider env vars, `AgentExecutor.execute()` options, CLI flags - [Deployment](docs/deployment.md) - `loushy build` targets: Node server, Docker, Cloudflare Workers - [API Overview](docs/api-overview.md) - the main exports; `npm run docs:build` generates the full TypeDoc reference +- [Tracing and observability](docs/observability.md) - OpenTelemetry GenAI spans, attribute table, content opt-in - [Testing](docs/testing.md) - unit-test agents deterministically with the scripted `mockModel` - [Skills](docs/skills.md) - on-demand instructions: `defineSkill()`, `loadSkills()`, how they save context - [Agent Forge](docs/agent-forge.md) - the visual dashboard (`loushy studio`): quickstart, first-agent walkthrough, hook authoring @@ -447,6 +448,12 @@ await AgentExecutor.execute({ }); ``` +Spans follow the OpenTelemetry GenAI semantic conventions (`invoke_agent`, +`chat {model}`, `execute_tool {tool}`; flows are traced too). Message and +tool-argument content is only recorded with `captureContent: true`. See +[Tracing and observability](docs/observability.md) for the attribute table, +the opt-in and the deprecated pre-GenAI names. + Ready-made exporters live in [examples/tracing](examples/tracing) (console and real OpenTelemetry bridges). diff --git a/apps/agent-forge/server/__tests__/runRegistry.test.ts b/apps/agent-forge/server/__tests__/runRegistry.test.ts index 650ceaee2..536c0d04c 100644 --- a/apps/agent-forge/server/__tests__/runRegistry.test.ts +++ b/apps/agent-forge/server/__tests__/runRegistry.test.ts @@ -143,13 +143,15 @@ describe('RunManager', () => { await runManager.run('agent-spans', 'please use current-date', SPEC); await waitForStatus(runManager, 'agent-spans', (s) => s.status === 'stopped'); - const runSpans = spans.filter((s) => s.name === 'agent.run'); + const runSpans = spans.filter((s) => s.attributes['gen_ai.operation.name'] === 'invoke_agent'); expect(runSpans).toHaveLength(2); // start + end const [startSpan, endSpan] = runSpans; expect(startSpan.endTime).toBeUndefined(); expect(endSpan.endTime).toBeGreaterThanOrEqual(endSpan.startTime); - expect(spans.some((s) => s.name === 'llm.generate')).toBe(true); - expect(spans.some((s) => s.name === 'tool.call' && s.parentId === startSpan.id)).toBe(true); + expect(spans.some((s) => s.attributes['gen_ai.operation.name'] === 'chat')).toBe(true); + expect( + spans.some((s) => s.attributes['gen_ai.operation.name'] === 'execute_tool' && s.parentId === startSpan.id) + ).toBe(true); }); it('pauses a run at a configured breakpoint and resumes on continueRun() (O3)', async () => { diff --git a/apps/agent-forge/src/components/debug/TracePanel.tsx b/apps/agent-forge/src/components/debug/TracePanel.tsx index c1737f2f5..f058220aa 100644 --- a/apps/agent-forge/src/components/debug/TracePanel.tsx +++ b/apps/agent-forge/src/components/debug/TracePanel.tsx @@ -6,8 +6,13 @@ import type { AgentGraphSpec } from '../../graph/types'; import { findLlmNodeId, findToolNodeId } from './nodeLookup'; function nodeIdForSpan(graph: AgentGraphSpec, span: SpanEvent): string | undefined { - if (span.name === 'llm.generate') return findLlmNodeId(graph); - if (span.name === 'tool.call') return findToolNodeId(graph, span.attributes.toolName as string | undefined); + // SDK spans follow the OpenTelemetry GenAI conventions (LOU-D9): the operation is + // `chat` (model call) or `execute_tool` (tool call). + const operation = span.attributes['gen_ai.operation.name']; + if (operation === 'chat') return findLlmNodeId(graph); + if (operation === 'execute_tool') { + return findToolNodeId(graph, span.attributes['gen_ai.tool.name'] as string | undefined); + } return undefined; } diff --git a/docs/api-overview.md b/docs/api-overview.md index 0eabb5aa6..cc73d3783 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -342,7 +342,9 @@ The built-in context windows and prices are a dated snapshot (see the retrieval `createOtelTraceExporter()` from the `@loushy/build-ai-agent/otel` subpath (requires the optional peer dependency `@opentelemetry/api`) instead of hand-rolling the OTel bridge - see - `examples/tracing/run-otel.ts`. + `examples/tracing/run-otel.ts`. Spans follow the OpenTelemetry GenAI + semantic conventions (flows are traced too); see + [observability](observability.md). - `NoopSandbox` / `SubprocessSandbox` - sandboxing for tools that opt in via `requiresSandbox`; guardrails such as `createCommandGuardrail()` and `secretScanGuardrail`. diff --git a/docs/configuration.md b/docs/configuration.md index 0a1f9691e..161cd9fb9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -175,7 +175,8 @@ dependency prints the exact `npm install` command. | `onEvent` | Callback for execution events (`start`, `tool-call`, `finish`, ...). | | `approvalStore`, `sessionId` | Human-in-the-loop approvals (see `resumeAfterApproval()`). | | `checkpointStore` | Persist/resume execution checkpoints. | -| `exporter` | A `TraceExporter` for tracing spans. | +| `exporter` | A `TraceExporter` for tracing spans (OpenTelemetry GenAI conventions, see [observability](observability.md)). | +| `captureContent`, `redactContent` | Record message/tool content on `gen_ai.*` span attributes (opt-in) / omit the deprecated content attributes. | | `onLLMRequest`, `onLLMResponse`, `onToolCall`, `onToolResult` | Observability hooks. | It resolves to an `ExecutionResult`: `{ text, messages, toolCalls, usage, diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 000000000..2f49a091a --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,180 @@ +# Tracing and observability + +`AgentExecutor.execute()` and `FlowExecutor.execute()` emit spans through a +`TraceExporter`. Span names and attributes follow the +[OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/), +so GenAI-aware observability backends can recognise them without custom +mapping. + +> **Stability.** The GenAI conventions are in `Development` status (not +> stable). Attribute names have changed between versions of the spec before +> (for example `gen_ai.system` became `gen_ai.provider.name`) and may change +> again. This SDK follows the spec as retrieved on 2026-10-01 from the +> [semantic-conventions-genai repository](https://github.com/open-telemetry/semantic-conventions-genai) +> (where the conventions now live). Pin your SDK version if dashboards depend +> on these names. + +## Quick start + +```ts +import { AgentExecutor, type TraceExporter } from '@loushy/build-ai-agent'; + +const exporter: TraceExporter = { + onSpanStart: (span) => console.log('[start]', span.kind, span.name), + onSpanEnd: (span) => console.log('[end]', span.name, span.status?.code ?? 'ok', span.attributes), +}; + +await AgentExecutor.execute({ agent, input, provider, toolRegistry, exporter }); +``` + +For real OpenTelemetry spans, import `createOtelTraceExporter()` from +`@loushy/build-ai-agent/otel` (needs the optional peer dependency +`@opentelemetry/api` and a registered `TracerProvider`). It carries span kind +and error status over to OpenTelemetry. See `examples/tracing` +(`npm run example:tracing:console`, `npm run example:tracing:otel`). + +## Span tree + +``` +invoke_agent {agent name} INTERNAL + chat {model} CLIENT one per provider.generate() call + execute_tool {tool name} INTERNAL one per tool call +``` + +A flow run (`FlowExecutor.execute`) produces: + +``` +invoke_workflow {flow name} INTERNAL + flow.node {node type} INTERNAL one per node execution, nested like the flow + chat {model} CLIENT inside an llmCall node + execute_tool {tool name} INTERNAL inside a toolCall node +``` + +To nest a flow under an agent span, pass that span's id as `parentSpanId` +(`withSpan` hands the span to its callback); the flow span then becomes its +child. + +## Attributes + +Constants for every name live in `src/execution/semconv.ts` (exported as +`GenAiAttr`, `GenAiOperation`, `ErrorAttr`, `FlowAttr`, `LegacyAttr`). + +### Agent run: `invoke_agent {agent name}` + +| Attribute | Value | +| --- | --- | +| `gen_ai.operation.name` | `invoke_agent` | +| `gen_ai.agent.name` | The agent's `name` | +| `gen_ai.agent.id` | The agent's `id`, when set | +| `gen_ai.provider.name` | The provider's `name` | +| `gen_ai.conversation.id` | `sessionId`, when set | + +### Model call: `chat {model}` (CLIENT) + +| Attribute | Value | +| --- | --- | +| `gen_ai.operation.name` | `chat` | +| `gen_ai.provider.name` | The provider's `name` | +| `gen_ai.request.model` | The requested model | +| `gen_ai.request.temperature`, `gen_ai.request.max_tokens` | When set | +| `gen_ai.response.model` | When the provider reports it | +| `gen_ai.response.finish_reasons` | e.g. `["stop"]`, `["tool_call"]` | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | Token usage | + +### Tool call: `execute_tool {tool}` + +| Attribute | Value | +| --- | --- | +| `gen_ai.operation.name` | `execute_tool` | +| `gen_ai.tool.name` | The tool name | +| `gen_ai.tool.call.id` | The model's tool-call id (agent runs) | +| `gen_ai.tool.description` | The tool's description, when it has one | +| `gen_ai.tool.type` | `function` | +| `gen_ai.agent.name`, `gen_ai.conversation.id` | The calling agent / session | + +### Flows + +The GenAI spec defines no convention for flow nodes, so node spans use the +`loushy.flow.*` namespace. The run span uses the spec's workflow convention. + +| Span | Attribute | Value | +| --- | --- | --- | +| `invoke_workflow {name}` | `gen_ai.operation.name` | `invoke_workflow` | +| | `gen_ai.workflow.name` | The flow's `name` | +| | `loushy.flow.code` | The flow's `code` | +| | `loushy.flow.outcome` | `success` or `error` | +| `flow.node {type}` | `loushy.flow.node.id` | The node's id | +| | `loushy.flow.node.type` | The node type (`sequence`, `llmCall`, ...) | +| | `loushy.flow.outcome` | `success` or `error` | + +### Errors + +A failed span gets `error.type` and an error span status (OpenTelemetry +`ERROR`, message = the error message). `error.type` is the thrown error's +`name` (`_OTHER` when a non-`Error` is thrown). A tool call that returns an +error to the model (`isError`) is marked `error.type = tool_error`. + +## Message and argument content is opt-in + +Prompts, model output and tool arguments/results are sensitive and large, so +the `gen_ai.*` content attributes are **never recorded by default**. Opt in +per run: + +```ts +import { AgentExecutor, type TraceExporter } from '@loushy/build-ai-agent'; + +declare const exporter: TraceExporter; + +await AgentExecutor.execute({ agent, input, provider, exporter, captureContent: true }); +``` + +Flows take the same flag on their context (`captureContent`). When +`captureContent` is not set, the `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` +environment variable (the name the spec uses as its example) turns it on. +An explicit `captureContent: false` always wins. + +With it on, content is recorded as JSON strings (the spec's fallback for +spans) in the spec's message schema: + +| Attribute | On | Content | +| --- | --- | --- | +| `gen_ai.input.messages` | `invoke_agent`, `chat` | `[{ role, parts: [{ type: 'text', content }, { type: 'tool_call', id, name, arguments }, { type: 'tool_call_response', id, response }] }]` | +| `gen_ai.system_instructions` | `chat` | `[{ type: 'text', content }]` | +| `gen_ai.output.messages` | `chat` | `[{ role: 'assistant', parts: [...] }]` | +| `gen_ai.tool.call.arguments` | `execute_tool` | The arguments as JSON | +| `gen_ai.tool.call.result` | `execute_tool` | The result as JSON (not set when the tool failed) | + +## Deprecated attribute names + +Before the GenAI conventions, spans used ad-hoc names and the span names +`agent.run`, `llm.generate` and `tool.call`. The old attribute names are still +emitted next to the new ones so existing dashboards keep working, but they are +**deprecated** and will be removed in a future major version. Span names +changed (to what the spec requires); update queries that matched on them. + +| Deprecated | Use instead | +| --- | --- | +| span `agent.run` | span `invoke_agent {name}`, `gen_ai.operation.name = invoke_agent` | +| span `llm.generate` | span `chat {model}`, `gen_ai.operation.name = chat` | +| span `tool.call` | span `execute_tool {tool}`, `gen_ai.operation.name = execute_tool` | +| `model` | `gen_ai.request.model` | +| `promptTokens` | `gen_ai.usage.input_tokens` | +| `completionTokens` | `gen_ai.usage.output_tokens` | +| `totalTokens` | (sum the two usage attributes) | +| `finishReason` | `gen_ai.response.finish_reasons` (an array; `tool_calls` is now `tool_call`) | +| `toolName` | `gen_ai.tool.name` | +| `error` (message on a thrown error; boolean on `execute_tool`) | `error.type` and the span status | +| `input`, `prompt`, `args`, `result` | `gen_ai.input.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` (opt-in, see above) | + +The deprecated content attributes (`input`, `prompt`, `args`, `result`) keep +their old behavior: they are recorded unless `redactContent: true`. Set +`redactContent: true` if you want no content on spans unless you opt in with +`captureContent`. + +## Backends + +Any backend that ingests OpenTelemetry traces (OTLP) receives these spans. +Backends and tools that understand the GenAI semantic conventions can render +them as LLM, agent and tool calls with token usage; others show them as +ordinary spans with attributes. Which conventions a given backend supports, +and which version of them, varies; check its documentation. diff --git a/examples/tracing/console-exporter.ts b/examples/tracing/console-exporter.ts index 1ef7954a2..acd03f75d 100644 --- a/examples/tracing/console-exporter.ts +++ b/examples/tracing/console-exporter.ts @@ -12,7 +12,7 @@ import { Span, TraceExporter } from '../../src/execution/tracing'; export function createConsoleExporter(): TraceExporter { - // Tracks depth so nested spans (llm.generate/tool.call under agent.run) + // Tracks depth so nested spans (chat/execute_tool under invoke_agent) // print indented, purely as a readability aid. const depthById = new Map(); diff --git a/llms-full.txt b/llms-full.txt index f257a71ab..b02923f09 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -210,6 +210,7 @@ console.log(result.usage.totalTokens, result.finishReason, result.steps); - [Configuration](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/configuration.md) - agent spec fields, provider env vars, `AgentExecutor.execute()` options, CLI flags - [Deployment](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/deployment.md) - `loushy build` targets: Node server, Docker, Cloudflare Workers - [API Overview](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/api-overview.md) - the main exports; `npm run docs:build` generates the full TypeDoc reference +- [Tracing and observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md) - OpenTelemetry GenAI spans, attribute table, content opt-in - [Testing](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md) - unit-test agents deterministically with the scripted `mockModel` - [Skills](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/skills.md) - on-demand instructions: `defineSkill()`, `loadSkills()`, how they save context - [Agent Forge](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/agent-forge.md) - the visual dashboard (`loushy studio`): quickstart, first-agent walkthrough, hook authoring @@ -445,6 +446,12 @@ await AgentExecutor.execute({ }); ``` +Spans follow the OpenTelemetry GenAI semantic conventions (`invoke_agent`, +`chat {model}`, `execute_tool {tool}`; flows are traced too). Message and +tool-argument content is only recorded with `captureContent: true`. See +[Tracing and observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md) for the attribute table, +the opt-in and the deprecated pre-GenAI names. + Ready-made exporters live in [examples/tracing](https://github.com/LinuxDevil/agent-sdk/blob/main/examples/tracing) (console and real OpenTelemetry bridges). @@ -1351,7 +1358,9 @@ The built-in context windows and prices are a dated snapshot (see the retrieval `createOtelTraceExporter()` from the `@loushy/build-ai-agent/otel` subpath (requires the optional peer dependency `@opentelemetry/api`) instead of hand-rolling the OTel bridge - see - `examples/tracing/run-otel.ts`. + `examples/tracing/run-otel.ts`. Spans follow the OpenTelemetry GenAI + semantic conventions (flows are traced too); see + [observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md). - `NoopSandbox` / `SubprocessSandbox` - sandboxing for tools that opt in via `requiresSandbox`; guardrails such as `createCommandGuardrail()` and `secretScanGuardrail`. @@ -1966,7 +1975,8 @@ dependency prints the exact `npm install` command. | `onEvent` | Callback for execution events (`start`, `tool-call`, `finish`, ...). | | `approvalStore`, `sessionId` | Human-in-the-loop approvals (see `resumeAfterApproval()`). | | `checkpointStore` | Persist/resume execution checkpoints. | -| `exporter` | A `TraceExporter` for tracing spans. | +| `exporter` | A `TraceExporter` for tracing spans (OpenTelemetry GenAI conventions, see [observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md)). | +| `captureContent`, `redactContent` | Record message/tool content on `gen_ai.*` span attributes (opt-in) / omit the deprecated content attributes. | | `onLLMRequest`, `onLLMResponse`, `onToolCall`, `onToolResult` | Observability hooks. | It resolves to an `ExecutionResult`: `{ text, messages, toolCalls, usage, @@ -2164,6 +2174,189 @@ Targets are `DeploymentAdapter` objects (`scaffold`, `build`, `describe`) registered by name with `registerAdapter()`; both are exported from the package root. +# Tracing and observability + +Source: docs/observability.md + +`AgentExecutor.execute()` and `FlowExecutor.execute()` emit spans through a +`TraceExporter`. Span names and attributes follow the +[OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/), +so GenAI-aware observability backends can recognise them without custom +mapping. + +> **Stability.** The GenAI conventions are in `Development` status (not +> stable). Attribute names have changed between versions of the spec before +> (for example `gen_ai.system` became `gen_ai.provider.name`) and may change +> again. This SDK follows the spec as retrieved on 2026-10-01 from the +> [semantic-conventions-genai repository](https://github.com/open-telemetry/semantic-conventions-genai) +> (where the conventions now live). Pin your SDK version if dashboards depend +> on these names. + +## Quick start + +```ts +import { AgentExecutor, type TraceExporter } from '@loushy/build-ai-agent'; + +const exporter: TraceExporter = { + onSpanStart: (span) => console.log('[start]', span.kind, span.name), + onSpanEnd: (span) => console.log('[end]', span.name, span.status?.code ?? 'ok', span.attributes), +}; + +await AgentExecutor.execute({ agent, input, provider, toolRegistry, exporter }); +``` + +For real OpenTelemetry spans, import `createOtelTraceExporter()` from +`@loushy/build-ai-agent/otel` (needs the optional peer dependency +`@opentelemetry/api` and a registered `TracerProvider`). It carries span kind +and error status over to OpenTelemetry. See `examples/tracing` +(`npm run example:tracing:console`, `npm run example:tracing:otel`). + +## Span tree + +``` +invoke_agent {agent name} INTERNAL + chat {model} CLIENT one per provider.generate() call + execute_tool {tool name} INTERNAL one per tool call +``` + +A flow run (`FlowExecutor.execute`) produces: + +``` +invoke_workflow {flow name} INTERNAL + flow.node {node type} INTERNAL one per node execution, nested like the flow + chat {model} CLIENT inside an llmCall node + execute_tool {tool name} INTERNAL inside a toolCall node +``` + +To nest a flow under an agent span, pass that span's id as `parentSpanId` +(`withSpan` hands the span to its callback); the flow span then becomes its +child. + +## Attributes + +Constants for every name live in `src/execution/semconv.ts` (exported as +`GenAiAttr`, `GenAiOperation`, `ErrorAttr`, `FlowAttr`, `LegacyAttr`). + +### Agent run: `invoke_agent {agent name}` + +| Attribute | Value | +| --- | --- | +| `gen_ai.operation.name` | `invoke_agent` | +| `gen_ai.agent.name` | The agent's `name` | +| `gen_ai.agent.id` | The agent's `id`, when set | +| `gen_ai.provider.name` | The provider's `name` | +| `gen_ai.conversation.id` | `sessionId`, when set | + +### Model call: `chat {model}` (CLIENT) + +| Attribute | Value | +| --- | --- | +| `gen_ai.operation.name` | `chat` | +| `gen_ai.provider.name` | The provider's `name` | +| `gen_ai.request.model` | The requested model | +| `gen_ai.request.temperature`, `gen_ai.request.max_tokens` | When set | +| `gen_ai.response.model` | When the provider reports it | +| `gen_ai.response.finish_reasons` | e.g. `["stop"]`, `["tool_call"]` | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | Token usage | + +### Tool call: `execute_tool {tool}` + +| Attribute | Value | +| --- | --- | +| `gen_ai.operation.name` | `execute_tool` | +| `gen_ai.tool.name` | The tool name | +| `gen_ai.tool.call.id` | The model's tool-call id (agent runs) | +| `gen_ai.tool.description` | The tool's description, when it has one | +| `gen_ai.tool.type` | `function` | +| `gen_ai.agent.name`, `gen_ai.conversation.id` | The calling agent / session | + +### Flows + +The GenAI spec defines no convention for flow nodes, so node spans use the +`loushy.flow.*` namespace. The run span uses the spec's workflow convention. + +| Span | Attribute | Value | +| --- | --- | --- | +| `invoke_workflow {name}` | `gen_ai.operation.name` | `invoke_workflow` | +| | `gen_ai.workflow.name` | The flow's `name` | +| | `loushy.flow.code` | The flow's `code` | +| | `loushy.flow.outcome` | `success` or `error` | +| `flow.node {type}` | `loushy.flow.node.id` | The node's id | +| | `loushy.flow.node.type` | The node type (`sequence`, `llmCall`, ...) | +| | `loushy.flow.outcome` | `success` or `error` | + +### Errors + +A failed span gets `error.type` and an error span status (OpenTelemetry +`ERROR`, message = the error message). `error.type` is the thrown error's +`name` (`_OTHER` when a non-`Error` is thrown). A tool call that returns an +error to the model (`isError`) is marked `error.type = tool_error`. + +## Message and argument content is opt-in + +Prompts, model output and tool arguments/results are sensitive and large, so +the `gen_ai.*` content attributes are **never recorded by default**. Opt in +per run: + +```ts +import { AgentExecutor, type TraceExporter } from '@loushy/build-ai-agent'; + +declare const exporter: TraceExporter; + +await AgentExecutor.execute({ agent, input, provider, exporter, captureContent: true }); +``` + +Flows take the same flag on their context (`captureContent`). When +`captureContent` is not set, the `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` +environment variable (the name the spec uses as its example) turns it on. +An explicit `captureContent: false` always wins. + +With it on, content is recorded as JSON strings (the spec's fallback for +spans) in the spec's message schema: + +| Attribute | On | Content | +| --- | --- | --- | +| `gen_ai.input.messages` | `invoke_agent`, `chat` | `[{ role, parts: [{ type: 'text', content }, { type: 'tool_call', id, name, arguments }, { type: 'tool_call_response', id, response }] }]` | +| `gen_ai.system_instructions` | `chat` | `[{ type: 'text', content }]` | +| `gen_ai.output.messages` | `chat` | `[{ role: 'assistant', parts: [...] }]` | +| `gen_ai.tool.call.arguments` | `execute_tool` | The arguments as JSON | +| `gen_ai.tool.call.result` | `execute_tool` | The result as JSON (not set when the tool failed) | + +## Deprecated attribute names + +Before the GenAI conventions, spans used ad-hoc names and the span names +`agent.run`, `llm.generate` and `tool.call`. The old attribute names are still +emitted next to the new ones so existing dashboards keep working, but they are +**deprecated** and will be removed in a future major version. Span names +changed (to what the spec requires); update queries that matched on them. + +| Deprecated | Use instead | +| --- | --- | +| span `agent.run` | span `invoke_agent {name}`, `gen_ai.operation.name = invoke_agent` | +| span `llm.generate` | span `chat {model}`, `gen_ai.operation.name = chat` | +| span `tool.call` | span `execute_tool {tool}`, `gen_ai.operation.name = execute_tool` | +| `model` | `gen_ai.request.model` | +| `promptTokens` | `gen_ai.usage.input_tokens` | +| `completionTokens` | `gen_ai.usage.output_tokens` | +| `totalTokens` | (sum the two usage attributes) | +| `finishReason` | `gen_ai.response.finish_reasons` (an array; `tool_calls` is now `tool_call`) | +| `toolName` | `gen_ai.tool.name` | +| `error` (message on a thrown error; boolean on `execute_tool`) | `error.type` and the span status | +| `input`, `prompt`, `args`, `result` | `gen_ai.input.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` (opt-in, see above) | + +The deprecated content attributes (`input`, `prompt`, `args`, `result`) keep +their old behavior: they are recorded unless `redactContent: true`. Set +`redactContent: true` if you want no content on spans unless you opt in with +`captureContent`. + +## Backends + +Any backend that ingests OpenTelemetry traces (OTLP) receives these spans. +Backends and tools that understand the GenAI semantic conventions can render +them as LLM, agent and tool calls with token usage; others show them as +ordinary spans with attributes. Which conventions a given backend supports, +and which version of them, varies; check its documentation. + # Skills Source: docs/skills.md diff --git a/llms.txt b/llms.txt index ad031d9a7..b34ea3137 100644 --- a/llms.txt +++ b/llms.txt @@ -9,6 +9,7 @@ - [Quick Start](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/quick-start.md): Every TypeScript snippet on this page is a complete, standalone ES module (`.mts`). - [API Overview](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/api-overview.md): The package root (`@loushy/build-ai-agent`) exports everything below. - [Configuration](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/configuration.md): A declarative agent is a YAML (`.yaml`/`.yml`) or JSON (`.json`) file, validated with zod by `loadSpec()` (`src/spec/schema.ts`). +- [Tracing and observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md): `AgentExecutor.execute()` and `FlowExecutor.execute()` emit spans through a `TraceExporter`. - [Skills](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/skills.md): A skill is a named chunk of instructions (a markdown document) that the model loads only when it needs it. - [Testing agents](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md): `@loushy/build-ai-agent/testing` ships `mockModel`: a scripted, deterministic `LLMProvider` for unit tests. diff --git a/src/execution/AgentExecutor.test.ts b/src/execution/AgentExecutor.test.ts index 945b011d6..a28298df4 100644 --- a/src/execution/AgentExecutor.test.ts +++ b/src/execution/AgentExecutor.test.ts @@ -1237,7 +1237,7 @@ describe('AgentExecutor', () => { return mockProvider; } - it('produces a 3-level span tree: agent.run -> llm.generate, tool.call, with correct parent/child ids', async () => { + it('produces a 3-level span tree: invoke_agent -> chat, execute_tool, with correct parent/child ids', async () => { const { tool } = await import('ai'); const { z } = await import('zod'); @@ -1267,13 +1267,18 @@ describe('AgentExecutor', () => { }); const names = starts.map((s) => s.name); - expect(names).toEqual(['agent.run', 'llm.generate', 'tool.call', 'llm.generate']); + expect(names).toEqual([ + 'invoke_agent Test Agent', + expect.stringMatching(/^chat/), + 'execute_tool echoTool', + expect.stringMatching(/^chat/), + ]); expect(starts).toHaveLength(4); expect(ends).toHaveLength(4); - const agentRunSpan = starts.find((s) => s.name === 'agent.run')!; - const llmSpans = starts.filter((s) => s.name === 'llm.generate'); - const toolSpan = starts.find((s) => s.name === 'tool.call')!; + const agentRunSpan = starts.find((s) => s.name.startsWith('invoke_agent'))!; + const llmSpans = starts.filter((s) => s.name.startsWith('chat')); + const toolSpan = starts.find((s) => s.name.startsWith('execute_tool'))!; expect(agentRunSpan.parentId).toBeUndefined(); for (const llmSpan of llmSpans) { @@ -1314,12 +1319,12 @@ describe('AgentExecutor', () => { const serialized = JSON.stringify(ends); expect(serialized).toContain('secret-value'); - const llmSpan = ends.find((s) => s.name === 'llm.generate'); + const llmSpan = ends.find((s) => s.name.startsWith('chat')); expect(llmSpan?.attributes.prompt).toBeDefined(); expect(llmSpan?.attributes.promptTokens).toBeDefined(); expect(llmSpan?.attributes.finishReason).toBeDefined(); - const toolSpan = ends.find((s) => s.name === 'tool.call'); + const toolSpan = ends.find((s) => s.name.startsWith('execute_tool')); expect(toolSpan?.attributes.args).toBeDefined(); expect(toolSpan?.attributes.result).toBeDefined(); expect(toolSpan?.attributes.error).toBe(false); @@ -1359,12 +1364,12 @@ describe('AgentExecutor', () => { const serialized = JSON.stringify(ends); expect(serialized).not.toContain('secret-value'); - const llmSpan = ends.find((s) => s.name === 'llm.generate'); + const llmSpan = ends.find((s) => s.name.startsWith('chat')); expect(llmSpan?.attributes.prompt).toBeUndefined(); expect(llmSpan?.attributes.promptTokens).toBeDefined(); expect(llmSpan?.attributes.finishReason).toBeDefined(); - const toolSpan = ends.find((s) => s.name === 'tool.call'); + const toolSpan = ends.find((s) => s.name.startsWith('execute_tool')); expect(toolSpan?.attributes.args).toBeUndefined(); expect(toolSpan?.attributes.result).toBeUndefined(); expect(toolSpan?.attributes.error).toBe(false); diff --git a/src/execution/AgentExecutor.ts b/src/execution/AgentExecutor.ts index 555d6b0b7..8885d7f5e 100644 --- a/src/execution/AgentExecutor.ts +++ b/src/execution/AgentExecutor.ts @@ -13,6 +13,12 @@ import { SandboxAdapter, NoopSandbox } from '../security/sandboxCore'; import { ApprovalStore, ExecutionSnapshot, PendingApproval } from './ApprovalGate'; import { CheckpointStore } from './checkpoint'; import { TraceExporter, withSpan } from './tracing'; +import { + agentRunSpanInit, + recordToolOutcome, + resolveCaptureContent, + toolSpanInit, +} from './genAiSpans'; import { HookRegistry } from './hooks'; import { isAbortError } from './errors'; import { @@ -194,21 +200,32 @@ export interface ExecuteOptions { ) => void | Promise; /** * Trace exporter (LOU-E3/E4/E5). When provided, execute() wraps its run - * in a 3-level span tree: a top-level `agent.run` span, with a nested - * `llm.generate` span around each provider.generate() call and a nested - * `tool.call` span around each tool execution, both parented to - * `agent.run` via Span.parentId. Omitted (or no exporter) means + * in a 3-level span tree following the OpenTelemetry GenAI conventions + * (LOU-D9): an `invoke_agent {name}` span, with a nested `chat {model}` + * span around each provider.generate() call and a nested + * `execute_tool {tool}` span around each tool execution, both parented to + * the agent span via Span.parentId. Omitted (or no exporter) means * withSpan() is a no-op wrapper - execute()'s behavior is unchanged. */ exporter?: TraceExporter; /** - * When true, span attributes omit potentially sensitive content (the - * `agent.run` input isn't captured differently, but `llm.generate` - * leaves out `prompt` and `tool.call` leaves out `args`/`result`). - * Token counts, finish reason, tool name and error/latency are never - * redacted. Defaults to false. + * When true, the DEPRECATED span attributes omit potentially sensitive + * content (`chat` leaves out `prompt` and `execute_tool` leaves out + * `args`/`result`; the agent span's `input` is not redacted). Token + * counts, finish reason, tool name and error/latency are never redacted. + * Defaults to false. The `gen_ai.*` content attributes are governed by + * `captureContent` instead. */ redactContent?: boolean; + /** + * Record message and tool-argument content on the OpenTelemetry GenAI + * attributes (`gen_ai.input.messages`, `gen_ai.output.messages`, + * `gen_ai.system_instructions`, `gen_ai.tool.call.arguments`, + * `gen_ai.tool.call.result`). Off by default; falls back to the + * `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` environment + * variable when omitted. + */ + captureContent?: boolean; /** * SandboxAdapter used for tools flagged `requiresSandbox` (LOU-F5). Since * AgentExecutor is a static, instance-free API, this is read per-call @@ -386,16 +403,21 @@ export class AgentExecutor { // then threaded as `parentId` into the nested 'llm.generate' and // 'tool.call' withSpan() calls, giving the 3-level span tree its // parent/child relationships without any instance state. + const init = agentRunSpanInit( + { ...options, input }, + resolveCaptureContent(options.captureContent) + ); return withSpan( exporter, - 'agent.run', - { input: typeof input === 'string' ? input : JSON.stringify(input) }, + init.name, + init.attributes, async (agentSpan) => this.runAgentLoop( { ...options, ...withSkills(options.agent, options.toolRegistry, options.skills) }, agentSpan.id ), - undefined + undefined, + init.kind ); } @@ -701,10 +723,11 @@ export class AgentExecutor { signal, } = options; + const init = toolSpanInit({ id: toolCall.id, name: toolCall.function.name }, { agent, toolRegistry, sessionId }); return withSpan( exporter, - 'tool.call', - { toolName: toolCall.function.name }, + init.name, + init.attributes, async (toolSpan) => { const toolCallStart = Date.now(); const executed = await this.executeToolCall( @@ -724,15 +747,20 @@ export class AgentExecutor { executed.args === undefined ? parseToolArguments(toolCall, toolCall.function.arguments) : executed.args; - toolSpan.attributes = { - ...toolSpan.attributes, - ...(redactContent ? {} : { args: parsedArgs, result: executed.result }), - error: !!executed.error, - latencyMs: Date.now() - toolCallStart, - }; + recordToolOutcome( + toolSpan, + { + args: parsedArgs, + result: executed.result, + error: executed.error, + latencyMs: Date.now() - toolCallStart, + }, + { redactContent, captureContent: resolveCaptureContent(options.captureContent) } + ); return executed; }, - agentSpanId + agentSpanId, + init.kind ); } diff --git a/src/execution/genAiSemconv.test.ts b/src/execution/genAiSemconv.test.ts new file mode 100644 index 000000000..8d4a5b615 --- /dev/null +++ b/src/execution/genAiSemconv.test.ts @@ -0,0 +1,390 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { AgentExecutor } from './AgentExecutor'; +import { Span, TraceExporter } from './tracing'; +import { CAPTURE_CONTENT_ENV } from './semconv'; +import { FlowExecutor } from '../flows/FlowExecutor'; +import { ToolRegistry, defineTool } from '../tools'; +import { AgentBuilder } from '../core'; +import { AgentType, AgentFlow } from '../types'; +import { mockModel } from '../testing'; + +/** In-memory exporter: remembers every span, in start order, in its final state. */ +function memoryExporter() { + const spans: Span[] = []; + const exporter: TraceExporter = { + onSpanStart: (span) => spans.push(span), + onSpanEnd: () => undefined, + }; + return { exporter, spans }; +} + +const weather = defineTool({ + name: 'get_weather', + description: 'Get the weather for a city', + input: z.object({ city: z.string() }), + execute: async ({ city }) => ({ city, tempC: 21 }), +}); + +function setup(tools: ToolRegistry = new ToolRegistry()) { + tools.register(weather); + const agent = { + ...AgentBuilder.create() + .setType(AgentType.SmartAssistant) + .setName('Weather Bot') + .addTool('get_weather', { tool: 'get_weather', options: {} }) + .build(), + id: 'agent-1', + prompt: 'You are terse.', + settings: { model: 'test-model' }, + }; + return { agent, toolRegistry: tools }; +} + +function modelWithOneToolCall() { + return mockModel([ + { + toolCalls: [{ id: 'call_abc', name: 'get_weather', args: { city: 'Paris' } }], + usage: { inputTokens: 10, outputTokens: 5 }, + }, + { text: 'It is 21C in Paris.', usage: { inputTokens: 20, outputTokens: 7 } }, + ]); +} + +async function runAgent(extra: { captureContent?: boolean; redactContent?: boolean } = {}) { + const { agent, toolRegistry } = setup(); + const { exporter, spans } = memoryExporter(); + const provider = modelWithOneToolCall(); + await AgentExecutor.execute({ + agent, + input: 'Weather in Paris?', + provider, + toolRegistry, + exporter, + sessionId: 'session-9', + temperature: 0.2, + maxTokens: 256, + ...extra, + }); + return { spans, provider }; +} + +describe('OpenTelemetry GenAI semantic conventions (LOU-D9)', () => { + afterEach(() => { + delete process.env[CAPTURE_CONTENT_ENV]; + }); + + it('names spans and sets kinds per the invoke_agent / chat / execute_tool conventions', async () => { + const { spans } = await runAgent(); + + expect(spans.map((s) => [s.name, s.kind])).toEqual([ + ['invoke_agent Weather Bot', 'internal'], + ['chat test-model', 'client'], + ['execute_tool get_weather', 'internal'], + ['chat test-model', 'client'], + ]); + const [agentSpan, ...children] = spans; + expect(children.every((s) => s.parentId === agentSpan.id)).toBe(true); + }); + + it('records the exact agent-run attribute keys and values', async () => { + const { spans, provider } = await runAgent(); + + expect(spans[0].attributes).toEqual({ + 'gen_ai.operation.name': 'invoke_agent', + 'gen_ai.agent.name': 'Weather Bot', + 'gen_ai.agent.id': 'agent-1', + 'gen_ai.provider.name': provider.name, + 'gen_ai.conversation.id': 'session-9', + input: 'Weather in Paris?', // deprecated + }); + }); + + it('records the exact model-call attribute keys and values (dual-emitting the deprecated names)', async () => { + const { spans, provider } = await runAgent(); + + expect(spans[1].attributes).toMatchObject({ + 'gen_ai.operation.name': 'chat', + 'gen_ai.provider.name': provider.name, + 'gen_ai.request.model': 'test-model', + 'gen_ai.request.temperature': 0.2, + 'gen_ai.request.max_tokens': 256, + 'gen_ai.response.finish_reasons': ['tool_call'], + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 5, + }); + expect(Object.keys(spans[1].attributes).sort()).toEqual( + [ + 'gen_ai.operation.name', + 'gen_ai.provider.name', + 'gen_ai.request.model', + 'gen_ai.request.temperature', + 'gen_ai.request.max_tokens', + 'gen_ai.response.finish_reasons', + 'gen_ai.usage.input_tokens', + 'gen_ai.usage.output_tokens', + // deprecated, still emitted + 'model', + 'prompt', + 'promptTokens', + 'completionTokens', + 'totalTokens', + 'finishReason', + ].sort() + ); + expect(spans[1].attributes).toMatchObject({ + model: 'test-model', + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + finishReason: 'tool_calls', + }); + expect(spans[3].attributes['gen_ai.response.finish_reasons']).toEqual(['stop']); + }); + + it('records the exact tool-execution attribute keys and values', async () => { + const { spans } = await runAgent(); + + expect(spans[2].attributes).toEqual({ + 'gen_ai.operation.name': 'execute_tool', + 'gen_ai.tool.name': 'get_weather', + 'gen_ai.tool.call.id': 'call_abc', + 'gen_ai.tool.description': 'Get the weather for a city', + 'gen_ai.tool.type': 'function', + 'gen_ai.agent.name': 'Weather Bot', + 'gen_ai.conversation.id': 'session-9', + // deprecated + toolName: 'get_weather', + args: { city: 'Paris' }, + result: { city: 'Paris', tempC: 21 }, + error: false, + latencyMs: expect.any(Number), + }); + expect(spans[2].status).toBeUndefined(); + }); + + it('never records gen_ai content attributes by default', async () => { + const { spans } = await runAgent(); + + const contentKeys = [ + 'gen_ai.input.messages', + 'gen_ai.output.messages', + 'gen_ai.system_instructions', + 'gen_ai.tool.call.arguments', + 'gen_ai.tool.call.result', + ]; + for (const span of spans) { + for (const key of contentKeys) { + expect(span.attributes).not.toHaveProperty(key); + } + } + }); + + it('records content on the spec attributes when captureContent is true', async () => { + const { spans } = await runAgent({ captureContent: true }); + + expect(JSON.parse(spans[0].attributes['gen_ai.input.messages'] as string)).toEqual([ + { role: 'user', parts: [{ type: 'text', content: 'Weather in Paris?' }] }, + ]); + expect(JSON.parse(spans[1].attributes['gen_ai.system_instructions'] as string)).toEqual([ + { type: 'text', content: 'You are terse.' }, + ]); + expect(JSON.parse(spans[1].attributes['gen_ai.input.messages'] as string)).toEqual([ + { role: 'user', parts: [{ type: 'text', content: 'Weather in Paris?' }] }, + ]); + expect(JSON.parse(spans[1].attributes['gen_ai.output.messages'] as string)).toEqual([ + { + role: 'assistant', + parts: [{ type: 'tool_call', id: 'call_abc', name: 'get_weather', arguments: { city: 'Paris' } }], + }, + ]); + expect(JSON.parse(spans[2].attributes['gen_ai.tool.call.arguments'] as string)).toEqual({ city: 'Paris' }); + expect(JSON.parse(spans[2].attributes['gen_ai.tool.call.result'] as string)).toEqual({ + city: 'Paris', + tempC: 21, + }); + // The follow-up model call sees the tool result as a tool_call_response part. + const followUp = JSON.parse(spans[3].attributes['gen_ai.input.messages'] as string); + expect(followUp.at(-1)).toEqual({ + role: 'tool', + parts: [{ type: 'tool_call_response', id: 'call_abc', response: { city: 'Paris', tempC: 21 } }], + }); + }); + + it('honors the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT env var, which captureContent overrides', async () => { + process.env[CAPTURE_CONTENT_ENV] = 'true'; + const viaEnv = await runAgent(); + expect(viaEnv.spans[1].attributes).toHaveProperty('gen_ai.input.messages'); + + const overridden = await runAgent({ captureContent: false }); + expect(overridden.spans[1].attributes).not.toHaveProperty('gen_ai.input.messages'); + }); + + it('records error.type and an error status when the model call throws', async () => { + const { agent, toolRegistry } = setup(); + const { exporter, spans } = memoryExporter(); + const provider = mockModel([]); + + await expect( + AgentExecutor.execute({ agent, input: 'hi', provider, toolRegistry, exporter }) + ).rejects.toThrow(); + + const chat = spans.find((s) => s.name.startsWith('chat'))!; + expect(chat.attributes['error.type']).toEqual(expect.any(String)); + expect(chat.attributes['error.type']).not.toBe(''); + expect(chat.status).toMatchObject({ code: 'error' }); + expect(spans[0].status).toMatchObject({ code: 'error' }); + expect(spans[0].attributes['error.type']).toEqual(expect.any(String)); + }); + + it('records error.type and an error status when a tool fails', async () => { + const failing = defineTool({ + name: 'get_weather', + description: 'Always fails', + input: z.object({ city: z.string() }), + execute: async () => { + throw new TypeError('upstream down'); + }, + }); + const registry = new ToolRegistry(); + registry.register(failing); + const { agent } = setup(new ToolRegistry()); + const { exporter, spans } = memoryExporter(); + + await AgentExecutor.execute({ + agent, + input: 'Weather in Paris?', + provider: modelWithOneToolCall(), + toolRegistry: registry, + exporter, + }); + + const tool = spans.find((s) => s.name === 'execute_tool get_weather')!; + expect(tool.attributes['error.type']).toBe('tool_error'); + expect(tool.attributes.error).toBe(true); + expect(tool.status?.code).toBe('error'); + expect(tool.attributes).not.toHaveProperty('gen_ai.tool.call.result'); + }); + + it('still honors redactContent for the deprecated content attributes', async () => { + const { spans } = await runAgent({ redactContent: true }); + + expect(spans[1].attributes).not.toHaveProperty('prompt'); + expect(spans[2].attributes).not.toHaveProperty('args'); + expect(spans[2].attributes).not.toHaveProperty('result'); + expect(spans[1].attributes['gen_ai.usage.input_tokens']).toBe(10); + }); +}); + +describe('traced flows (LOU-D9)', () => { + const flow: AgentFlow = { + code: 'weather-flow', + name: 'Weather Flow', + flow: { + type: 'sequence', + id: 'root', + steps: [ + { type: 'toolCall', id: 'lookup', tool: 'get_weather', arguments: { city: 'Paris' }, outputVariable: 'w' }, + { type: 'llmCall', id: 'summarize', prompt: 'Summarize {{w}}', outputVariable: 'summary' }, + { type: 'return', id: 'done', value: '$summary' }, + ], + }, + } as AgentFlow; + + function flowContext(extra: object = {}) { + const { agent, toolRegistry } = setup(); + const { exporter, spans } = memoryExporter(); + const provider = mockModel([{ text: 'Sunny.', usage: { inputTokens: 3, outputTokens: 1 } }]); + return { + spans, + context: { agent, provider, toolRegistry, variables: {}, exporter, ...extra }, + }; + } + + it('wraps the run and each node in spans, parenting model and tool spans under their node', async () => { + const { spans, context } = flowContext(); + + const result = await FlowExecutor.execute(flow, context); + + expect(result.success).toBe(true); + expect(spans.map((s) => s.name)).toEqual([ + 'invoke_workflow Weather Flow', + 'flow.node sequence', + 'flow.node toolCall', + 'execute_tool get_weather', + 'flow.node llmCall', + 'chat test-model', + 'flow.node return', + ]); + const byName = (name: string) => spans.find((s) => s.name === name)!; + const run = byName('invoke_workflow Weather Flow'); + const sequence = byName('flow.node sequence'); + expect(run.parentId).toBeUndefined(); + expect(sequence.parentId).toBe(run.id); + expect(byName('flow.node toolCall').parentId).toBe(sequence.id); + expect(byName('flow.node llmCall').parentId).toBe(sequence.id); + expect(byName('flow.node return').parentId).toBe(sequence.id); + expect(byName('execute_tool get_weather').parentId).toBe(byName('flow.node toolCall').id); + expect(byName('chat test-model').parentId).toBe(byName('flow.node llmCall').id); + expect(byName('chat test-model').kind).toBe('client'); + }); + + it('records flow and node attributes and outcomes', async () => { + const { spans, context } = flowContext(); + + await FlowExecutor.execute(flow, context); + + expect(spans[0].attributes).toEqual({ + 'gen_ai.operation.name': 'invoke_workflow', + 'gen_ai.workflow.name': 'Weather Flow', + 'loushy.flow.code': 'weather-flow', + 'loushy.flow.outcome': 'success', + }); + expect(spans[2].attributes).toEqual({ + 'loushy.flow.node.id': 'lookup', + 'loushy.flow.node.type': 'toolCall', + 'loushy.flow.outcome': 'success', + }); + expect(spans[5].attributes['gen_ai.usage.input_tokens']).toBe(3); + }); + + it('nests the run under a parent span when parentSpanId is given', async () => { + const { spans, context } = flowContext({ parentSpanId: 'agent-span-1' }); + + await FlowExecutor.execute(flow, context); + + expect(spans[0].parentId).toBe('agent-span-1'); + }); + + it('marks failed nodes and the run with error outcome, error.type and error status', async () => { + const failing = { + code: 'bad', + name: 'Bad Flow', + flow: { type: 'sequence', id: 'root', steps: [{ type: 'throw', id: 'boom', message: 'nope' }] }, + } as AgentFlow; + const { spans, context } = flowContext(); + + const result = await FlowExecutor.execute(failing, context); + + expect(result.success).toBe(false); + const run = spans[0]; + const boom = spans.find((s) => s.name === 'flow.node throw')!; + for (const span of [run, boom]) { + expect(span.attributes['loushy.flow.outcome']).toBe('error'); + expect(span.attributes['error.type']).toBe('Error'); + expect(span.status).toEqual({ code: 'error', message: 'nope' }); + } + }); + + it('records no content attributes unless captureContent is set', async () => { + const off = flowContext(); + await FlowExecutor.execute(flow, off.context); + expect(off.spans[5].attributes).not.toHaveProperty('gen_ai.input.messages'); + expect(off.spans[3].attributes).not.toHaveProperty('gen_ai.tool.call.arguments'); + + const on = flowContext({ captureContent: true }); + await FlowExecutor.execute(flow, on.context); + expect(on.spans[5].attributes).toHaveProperty('gen_ai.input.messages'); + expect(on.spans[3].attributes['gen_ai.tool.call.arguments']).toBe(JSON.stringify({ city: 'Paris' })); + }); +}); diff --git a/src/execution/genAiSpans.ts b/src/execution/genAiSpans.ts new file mode 100644 index 000000000..9dd864329 --- /dev/null +++ b/src/execution/genAiSpans.ts @@ -0,0 +1,227 @@ +/** + * Builds the span names, kinds and attributes for agent runs, model calls + * and tool executions following the OpenTelemetry GenAI semantic + * conventions (LOU-D9), dual-emitting the SDK's older ad-hoc attribute + * names. See `semconv.ts` for the spec URL, retrieval date and stability. + * + * Message and argument CONTENT is sensitive: the `gen_ai.*` content + * attributes are only recorded when `captureContent` is true (or the + * `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` environment variable + * is `true` and `captureContent` was not set). The deprecated content + * attributes (`input`, `prompt`, `args`, `result`) keep their previous + * behavior: recorded unless `redactContent` is true. + */ + +import type { GenerateOptions, GenerateResult, LLMProvider, Message, ToolCall } from '../providers'; +import type { ToolRegistry } from '../tools'; +import type { AgentConfig } from '../types'; +import { ErrorAttr, GenAiAttr, GenAiOperation, LegacyAttr, TOOL_TYPE_FUNCTION, CAPTURE_CONTENT_ENV } from './semconv'; +import type { Span, SpanKind } from './tracing'; + +/** Name, kind and starting attributes for a span about to be opened. */ +interface SpanInit { + name: string; + kind: SpanKind; + attributes: Record; +} + +/** Content-privacy switches shared by every traced entry point. */ +interface ContentOptions { + /** + * Record message and tool-argument content on the `gen_ai.*` attributes + * (`gen_ai.input.messages`, `gen_ai.output.messages`, + * `gen_ai.system_instructions`, `gen_ai.tool.call.arguments`, + * `gen_ai.tool.call.result`). Off by default because content is sensitive + * and large; when omitted it falls back to the + * `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` environment + * variable. + * + * @example + * await AgentExecutor.execute({ agent, input, provider, exporter, captureContent: true }); + */ + captureContent?: boolean; + /** Omit the deprecated `prompt`/`args`/`result` attributes (default false). */ + redactContent?: boolean; +} + +/** Resolves `captureContent`, falling back to the OTel content env var. */ +export function resolveCaptureContent(captureContent: boolean | undefined): boolean { + if (captureContent !== undefined) { + return captureContent; + } + return typeof process !== 'undefined' && process.env?.[CAPTURE_CONTENT_ENV]?.toLowerCase() === 'true'; +} + +/** Drops `undefined` values so attribute maps contain only what is known. */ +export function defined(attributes: Record): Record { + return Object.fromEntries(Object.entries(attributes).filter(([, value]) => value !== undefined)); +} + +function textPart(content: string) { + return { type: 'text', content }; +} + +function parseJson(text: string): unknown { + try { + return JSON.parse(text); + } catch { + return text; + } +} + +function toolCallPart(call: ToolCall) { + return { + type: 'tool_call', + id: call.id, + name: call.function.name, + arguments: parseJson(call.function.arguments), + }; +} + +function genAiMessage(message: Message) { + if (message.role === 'tool') { + return { + role: 'tool', + parts: [{ type: 'tool_call_response', id: message.toolCallId, response: parseJson(message.content) }], + }; + } + const parts: unknown[] = message.content ? [textPart(message.content)] : []; + parts.push(...(message.toolCalls ?? []).map(toolCallPart)); + return { role: message.role, parts }; +} + +/** Content attributes for a model call's request (JSON strings, per spec fallback for spans). */ +function requestContent(messages: Message[]): Record { + const system = messages.filter((m) => m.role === 'system'); + const rest = messages.filter((m) => m.role !== 'system'); + return defined({ + [GenAiAttr.SYSTEM_INSTRUCTIONS]: + system.length > 0 ? JSON.stringify(system.map((m) => textPart(m.content))) : undefined, + [GenAiAttr.INPUT_MESSAGES]: JSON.stringify(rest.map(genAiMessage)), + }); +} + +/** `invoke_agent {name}` span for one agent run. */ +export function agentRunSpanInit( + options: { agent: AgentConfig; provider?: LLMProvider; sessionId?: string; input: unknown }, + captureContent: boolean +): SpanInit { + const { agent, provider, sessionId, input } = options; + const text = typeof input === 'string' ? input : JSON.stringify(input); + return { + name: `${GenAiOperation.INVOKE_AGENT} ${agent.name}`, + kind: 'internal', + attributes: defined({ + [GenAiAttr.OPERATION_NAME]: GenAiOperation.INVOKE_AGENT, + [GenAiAttr.AGENT_NAME]: agent.name, + [GenAiAttr.AGENT_ID]: agent.id, + [GenAiAttr.PROVIDER_NAME]: provider?.name, + [GenAiAttr.CONVERSATION_ID]: sessionId, + [LegacyAttr.INPUT]: text, + [GenAiAttr.INPUT_MESSAGES]: captureContent + ? JSON.stringify([{ role: 'user', parts: [textPart(text)] }]) + : undefined, + }), + }; +} + +/** `chat {model}` span (CLIENT) for one provider.generate() call. */ +export function llmSpanInit( + provider: LLMProvider, + request: GenerateOptions, + content: ContentOptions & { captureContent: boolean } +): SpanInit { + return { + name: request.model ? `${GenAiOperation.CHAT} ${request.model}` : GenAiOperation.CHAT, + kind: 'client', + attributes: defined({ + [GenAiAttr.OPERATION_NAME]: GenAiOperation.CHAT, + [GenAiAttr.PROVIDER_NAME]: provider.name, + [GenAiAttr.REQUEST_MODEL]: request.model, + [GenAiAttr.REQUEST_TEMPERATURE]: request.temperature, + [GenAiAttr.REQUEST_MAX_TOKENS]: request.maxTokens, + [LegacyAttr.MODEL]: request.model, + ...(content.redactContent ? {} : { [LegacyAttr.PROMPT]: JSON.stringify(request.messages) }), + ...(content.captureContent ? requestContent(request.messages) : {}), + }), + }; +} + +/** The spec's `tool_call` finish reason is singular; ours is `tool_calls`. */ +function genAiFinishReason(reason: GenerateResult['finishReason']): string { + return reason === 'tool_calls' ? 'tool_call' : reason; +} + +function responseModel(generated: GenerateResult): string | undefined { + const raw = generated.rawResponse; + const model = raw?.response?.modelId ?? raw?.modelId ?? raw?.model; + return typeof model === 'string' ? model : undefined; +} + +/** Records usage, finish reason, response model (and output content) on a `chat` span. */ +export function recordLlmResult(span: Span, generated: GenerateResult, captureContent: boolean): void { + const parts: unknown[] = generated.text ? [textPart(generated.text)] : []; + parts.push(...(generated.toolCalls ?? []).map(toolCallPart)); + // Token counts and finish reason are never redacted. + span.attributes = { + ...span.attributes, + ...defined({ + [GenAiAttr.RESPONSE_MODEL]: responseModel(generated), + [GenAiAttr.RESPONSE_FINISH_REASONS]: [genAiFinishReason(generated.finishReason)], + [GenAiAttr.USAGE_INPUT_TOKENS]: generated.usage.promptTokens, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: generated.usage.completionTokens, + [LegacyAttr.PROMPT_TOKENS]: generated.usage.promptTokens, + [LegacyAttr.COMPLETION_TOKENS]: generated.usage.completionTokens, + [LegacyAttr.TOTAL_TOKENS]: generated.usage.totalTokens, + [LegacyAttr.FINISH_REASON]: generated.finishReason, + [GenAiAttr.OUTPUT_MESSAGES]: captureContent + ? JSON.stringify([{ role: 'assistant', parts }]) + : undefined, + }), + }; +} + +/** `execute_tool {name}` span for one tool execution. */ +export function toolSpanInit( + toolCall: { id?: string; name: string }, + context: { agent: AgentConfig; toolRegistry?: ToolRegistry; sessionId?: string } +): SpanInit { + const toolName = toolCall.name; + return { + name: `${GenAiOperation.EXECUTE_TOOL} ${toolName}`, + kind: 'internal', + attributes: defined({ + [GenAiAttr.OPERATION_NAME]: GenAiOperation.EXECUTE_TOOL, + [GenAiAttr.TOOL_NAME]: toolName, + [GenAiAttr.TOOL_CALL_ID]: toolCall.id, + [GenAiAttr.TOOL_DESCRIPTION]: context.toolRegistry?.get(toolName)?.tool?.description, + [GenAiAttr.TOOL_TYPE]: TOOL_TYPE_FUNCTION, + [GenAiAttr.AGENT_NAME]: context.agent.name, + [GenAiAttr.CONVERSATION_ID]: context.sessionId, + [LegacyAttr.TOOL_NAME]: toolName, + }), + }; +} + +/** Records args/result (content), error and latency on an `execute_tool` span. */ +export function recordToolOutcome( + span: Span, + outcome: { args: unknown; result: unknown; error?: string; latencyMs: number }, + content: ContentOptions & { captureContent: boolean } +): void { + span.attributes = { + ...span.attributes, + ...defined({ + ...(content.redactContent ? {} : { [LegacyAttr.ARGS]: outcome.args, [LegacyAttr.RESULT]: outcome.result }), + [LegacyAttr.ERROR]: !!outcome.error, + [LegacyAttr.LATENCY_MS]: outcome.latencyMs, + [ErrorAttr.TYPE]: outcome.error ? 'tool_error' : undefined, + [GenAiAttr.TOOL_CALL_ARGUMENTS]: content.captureContent ? JSON.stringify(outcome.args) : undefined, + [GenAiAttr.TOOL_CALL_RESULT]: + content.captureContent && !outcome.error ? JSON.stringify(outcome.result) : undefined, + }), + }; + if (outcome.error) { + span.status = { code: 'error', message: outcome.error }; + } +} diff --git a/src/execution/generateStep.ts b/src/execution/generateStep.ts index ed5ae2702..e17266e60 100644 --- a/src/execution/generateStep.ts +++ b/src/execution/generateStep.ts @@ -1,7 +1,7 @@ /** * The provider.generate() half of one AgentExecutor loop step: building the * request and tool definitions, the onLLMRequest/onLLMResponse callbacks, - * pre/post generate hooks, the `llm.generate` span, and LOU-T4's + * pre/post generate hooks, the `chat {model}` span, and LOU-T4's * compaction of a generate() failure. */ @@ -9,6 +9,7 @@ import { GenerateOptions, GenerateResult, Message, ToolDefinition } from '../pro import { AgentConfig } from '../types'; import { ToolRegistry } from '../tools'; import { withSpan } from './tracing'; +import { llmSpanInit, recordLlmResult, resolveCaptureContent } from './genAiSpans'; import { GenerateHookContext } from './hooks'; import { CompactedLLMProviderError, @@ -103,8 +104,8 @@ export async function prepareGenerateRequest( } /** - * Runs provider.generate() inside an `llm.generate` span parented to the - * run's `agent.run` span, followed by the onLLMResponse callback and + * Runs provider.generate() inside a `chat {model}` span parented to the + * run's `invoke_agent` span, followed by the onLLMResponse callback and * postGenerate hooks. */ export function generateInSpan( @@ -114,27 +115,19 @@ export function generateInSpan( agentSpanId: string ): Promise { const { provider, exporter, onLLMResponse, hooks, redactContent = false } = options; + const captureContent = resolveCaptureContent(options.captureContent); + const init = llmSpanInit(provider, generateRequest, { redactContent, captureContent }); return withSpan( exporter, - 'llm.generate', - { - model: generateRequest.model, - ...(redactContent ? {} : { prompt: JSON.stringify(generateRequest.messages) }), - }, + init.name, + init.attributes, async (llmSpan) => { const llmStart = Date.now(); const generated = await provider.generate(generateRequest); const llmLatencyMs = Date.now() - llmStart; - // Token counts and finish reason are never redacted. - llmSpan.attributes = { - ...llmSpan.attributes, - promptTokens: generated.usage.promptTokens, - completionTokens: generated.usage.completionTokens, - totalTokens: generated.usage.totalTokens, - finishReason: generated.finishReason, - }; + recordLlmResult(llmSpan, generated, captureContent); if (onLLMResponse) { await onLLMResponse(generated, llmLatencyMs); @@ -149,7 +142,8 @@ export function generateInSpan( return generated; }, - agentSpanId + agentSpanId, + init.kind ); } diff --git a/src/execution/index.ts b/src/execution/index.ts index 2a8c728e7..d1332aef1 100644 --- a/src/execution/index.ts +++ b/src/execution/index.ts @@ -13,6 +13,7 @@ export * from './ApprovalGate'; export * from './resume'; export * from './checkpoint'; export * from './tracing'; +export * from './semconv'; export * from './logger'; export * from './guardrails'; export * from './hooks'; diff --git a/src/execution/otel.test.ts b/src/execution/otel.test.ts index 2463686a9..2368ee5d1 100644 --- a/src/execution/otel.test.ts +++ b/src/execution/otel.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { Tracer, Span as OtelSpan, Context } from '@opentelemetry/api'; +import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { createOtelTraceExporter } from './otel'; import { Span } from './tracing'; @@ -151,3 +152,46 @@ describe('createOtelTraceExporter', () => { getTracerSpy.mockRestore(); }); }); + +describe('createOtelTraceExporter kind, status and array attributes (LOU-D9)', () => { + it('maps span kind client/internal to the OTel SpanKind on start', () => { + const fake = makeFakeTracer(); + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + exporter.onSpanStart(makeSpan({ id: 'c', kind: 'client' })); + exporter.onSpanStart(makeSpan({ id: 'i' })); + + const calls = (fake.tracer.startSpan as ReturnType).mock.calls; + expect(calls[0][1]).toMatchObject({ kind: SpanKind.CLIENT }); + expect(calls[1][1]).toMatchObject({ kind: SpanKind.INTERNAL }); + }); + + it('sets an ERROR status on end when the SDK span failed, and none otherwise', () => { + const fake = makeFakeTracer(); + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + const bad = vi.fn(); + const good = vi.fn(); + exporter.onSpanStart(makeSpan({ id: 'bad' })); + exporter.onSpanStart(makeSpan({ id: 'good' })); + (fake.startedSpans[0].span as { setStatus: unknown }).setStatus = bad; + (fake.startedSpans[1].span as { setStatus: unknown }).setStatus = good; + + exporter.onSpanEnd(makeSpan({ id: 'bad', endTime: 5, status: { code: 'error', message: 'boom' } })); + exporter.onSpanEnd(makeSpan({ id: 'good', endTime: 5 })); + + expect(bad).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR, message: 'boom' }); + expect(good).not.toHaveBeenCalled(); + }); + + it('passes homogeneous primitive arrays through and stringifies other arrays', () => { + const fake = makeFakeTracer(); + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + exporter.onSpanStart( + makeSpan({ attributes: { 'gen_ai.response.finish_reasons': ['stop'], mixed: ['a', 1], objs: [{ a: 1 }] } }) + ); + expect(fake.startedSpans[0].span.attributes).toEqual({ + 'gen_ai.response.finish_reasons': ['stop'], + mixed: JSON.stringify(['a', 1]), + objs: JSON.stringify([{ a: 1 }]), + }); + }); +}); diff --git a/src/execution/otel.ts b/src/execution/otel.ts index cf44f9b7c..29a8339ca 100644 --- a/src/execution/otel.ts +++ b/src/execution/otel.ts @@ -22,7 +22,16 @@ * time. */ -import { trace, context, Context, Span as OtelSpan, Tracer } from '@opentelemetry/api'; +import { + trace, + context, + AttributeValue, + Context, + Span as OtelSpan, + SpanKind as OtelSpanKind, + SpanStatusCode, + Tracer, +} from '@opentelemetry/api'; import { Span, TraceExporter } from './tracing'; /** @@ -52,8 +61,11 @@ export interface OtelTraceExporterOptions { * Creates a {@link TraceExporter} that starts/ends a real OpenTelemetry * span for every `Span` this SDK produces, propagating parent/child * relationships (`Span.parentId`) into OTel's context API so a real OTel - * backend sees the same `agent.run` -> `llm.generate`/`tool.call` span - * tree that `withSpan()` builds internally. + * backend sees the same `invoke_agent` -> `chat`/`execute_tool` span + * tree that `withSpan()` builds internally, with span kind and error status + * carried over. Span names and attributes follow the OpenTelemetry GenAI + * semantic conventions (see `semconv.ts`; message content is opt-in via + * `captureContent` on `AgentExecutor.execute()`/`FlowExecutor`). * * This only depends on `@opentelemetry/api` (the OTel instrumentation * API), not any concrete SDK/exporter backend - the host application is @@ -85,7 +97,11 @@ export function createOtelTraceExporter(options: OtelTraceExporterOptions = {}): const parentEntry = span.parentId ? otelSpansById.get(span.parentId) : undefined; const parentContext = parentEntry ? parentEntry.ctx : context.active(); - const otelSpan = tracer.startSpan(span.name, { startTime: span.startTime }, parentContext); + const otelSpan = tracer.startSpan( + span.name, + { startTime: span.startTime, kind: span.kind === 'client' ? OtelSpanKind.CLIENT : OtelSpanKind.INTERNAL }, + parentContext + ); // Seed OTel span attributes with whatever this SDK already knew at // span-start time (e.g. model, toolName). onSpanEnd below adds @@ -103,6 +119,9 @@ export function createOtelTraceExporter(options: OtelTraceExporterOptions = {}): } setAttributes(entry.span, span.attributes); + if (span.status?.code === 'error') { + entry.span.setStatus({ code: SpanStatusCode.ERROR, message: span.status.message }); + } entry.span.end(span.endTime); otelSpansById.delete(span.id); }, @@ -120,9 +139,21 @@ function setAttributes(otelSpan: OtelSpan, attributes: Record): // OTel attribute values must be a primitive or an array of a single // primitive type - stringify anything else (objects/arrays of // objects, e.g. `prompt`/`args`/`result`) so nothing throws. -function toOtelAttributeValue(value: unknown): string | number | boolean { +function toOtelAttributeValue(value: unknown): AttributeValue { if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { return value; } + if (isHomogeneousPrimitiveArray(value)) { + return value; + } return JSON.stringify(value); } + +// e.g. `gen_ai.response.finish_reasons` is a string[] attribute in the spec. +function isHomogeneousPrimitiveArray(value: unknown): value is string[] | number[] | boolean[] { + return ( + Array.isArray(value) && + value.length > 0 && + ['string', 'number', 'boolean'].some((type) => value.every((item) => typeof item === type)) + ); +} diff --git a/src/execution/semconv.ts b/src/execution/semconv.ts new file mode 100644 index 000000000..7752d59c1 --- /dev/null +++ b/src/execution/semconv.ts @@ -0,0 +1,100 @@ +/** + * OpenTelemetry GenAI semantic-convention names (LOU-D9). + * + * Every span name, attribute key and well-known value this SDK emits lives + * here so nothing is stringly-typed across files. + * + * Spec: https://opentelemetry.io/docs/specs/semconv/gen-ai/ - the GenAI + * conventions have moved to + * https://github.com/open-telemetry/semantic-conventions-genai + * (docs/gen-ai/gen-ai-spans.md and docs/gen-ai/gen-ai-agent-spans.md, + * schema model/gen-ai/*.json). Retrieved 2026-10-01 (main as of + * 2026-09-30, semconv v1.44.0 for the shared `error.*` attributes). + * + * STABILITY: the GenAI conventions are `Development` (not stable) as of + * that date. Attribute names have changed between versions before (for + * example `gen_ai.system` became `gen_ai.provider.name`), so they may change + * again. Pin the SDK version if you build dashboards on them. + */ + +/** `gen_ai.operation.name` well-known values this SDK emits. */ +export const GenAiOperation = { + /** Model call (inference span). */ + CHAT: 'chat', + /** One agent run. */ + INVOKE_AGENT: 'invoke_agent', + /** One tool execution. */ + EXECUTE_TOOL: 'execute_tool', + /** One flow run (`FlowExecutor.execute`). */ + INVOKE_WORKFLOW: 'invoke_workflow', +} as const; + +/** OpenTelemetry GenAI attribute keys (all `Development` stability). */ +export const GenAiAttr = { + OPERATION_NAME: 'gen_ai.operation.name', + PROVIDER_NAME: 'gen_ai.provider.name', + CONVERSATION_ID: 'gen_ai.conversation.id', + AGENT_ID: 'gen_ai.agent.id', + AGENT_NAME: 'gen_ai.agent.name', + WORKFLOW_NAME: 'gen_ai.workflow.name', + REQUEST_MODEL: 'gen_ai.request.model', + REQUEST_TEMPERATURE: 'gen_ai.request.temperature', + REQUEST_MAX_TOKENS: 'gen_ai.request.max_tokens', + RESPONSE_MODEL: 'gen_ai.response.model', + RESPONSE_FINISH_REASONS: 'gen_ai.response.finish_reasons', + USAGE_INPUT_TOKENS: 'gen_ai.usage.input_tokens', + USAGE_OUTPUT_TOKENS: 'gen_ai.usage.output_tokens', + TOOL_NAME: 'gen_ai.tool.name', + TOOL_CALL_ID: 'gen_ai.tool.call.id', + TOOL_DESCRIPTION: 'gen_ai.tool.description', + TOOL_TYPE: 'gen_ai.tool.type', + /** Opt-in content (see `captureContent`). */ + SYSTEM_INSTRUCTIONS: 'gen_ai.system_instructions', + INPUT_MESSAGES: 'gen_ai.input.messages', + OUTPUT_MESSAGES: 'gen_ai.output.messages', + TOOL_CALL_ARGUMENTS: 'gen_ai.tool.call.arguments', + TOOL_CALL_RESULT: 'gen_ai.tool.call.result', +} as const; + +/** `error.type` (stable in semconv) and its `_OTHER` fallback. */ +export const ErrorAttr = { + TYPE: 'error.type', + /** Fallback value when the thrown value is not an `Error`. */ + OTHER: '_OTHER', +} as const; + +/** `gen_ai.tool.type` value for tools executed by this SDK's host process. */ +export const TOOL_TYPE_FUNCTION = 'function'; + +/** + * The attribute names this SDK emitted before LOU-D9. They are still emitted + * alongside the GenAI ones (dual-emit) and are DEPRECATED. + */ +export const LegacyAttr = { + INPUT: 'input', + MODEL: 'model', + PROMPT: 'prompt', + PROMPT_TOKENS: 'promptTokens', + COMPLETION_TOKENS: 'completionTokens', + TOTAL_TOKENS: 'totalTokens', + FINISH_REASON: 'finishReason', + TOOL_NAME: 'toolName', + ARGS: 'args', + RESULT: 'result', + ERROR: 'error', + LATENCY_MS: 'latencyMs', +} as const; + +/** Attributes for flow spans (no GenAI convention exists for flow nodes). */ +export const FlowAttr = { + CODE: 'loushy.flow.code', + NODE_ID: 'loushy.flow.node.id', + NODE_TYPE: 'loushy.flow.node.type', + OUTCOME: 'loushy.flow.outcome', +} as const; + +/** Span-name prefix for a flow node span (`flow.node sequence`, ...). */ +export const FLOW_NODE_SPAN_NAME = 'flow.node'; + +/** Environment variable that turns content capture on when `captureContent` is not set. */ +export const CAPTURE_CONTENT_ENV = 'OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT'; diff --git a/src/execution/tracing.test.ts b/src/execution/tracing.test.ts index 631c7120b..5e992f49c 100644 --- a/src/execution/tracing.test.ts +++ b/src/execution/tracing.test.ts @@ -75,3 +75,35 @@ describe('withSpan', () => { expect(exporter.onSpanStart).toHaveBeenCalledTimes(2); }); }); + +describe('withSpan kind and error status (LOU-D9)', () => { + it('records the span kind when given and leaves it unset otherwise', async () => { + const { exporter, starts } = createSpyExporter(); + await withSpan(exporter, 'a', {}, async () => 1, undefined, 'client'); + await withSpan(exporter, 'b', {}, async () => 2); + expect(starts[0].kind).toBe('client'); + expect(starts[1].kind).toBeUndefined(); + }); + + it('sets error.type (the error name) and an error status when fn throws', async () => { + const { exporter, ends } = createSpyExporter(); + await expect( + withSpan(exporter, 's', {}, async () => { + throw new TypeError('bad'); + }) + ).rejects.toThrow('bad'); + expect(ends[0].attributes).toMatchObject({ error: 'bad', 'error.type': 'TypeError' }); + expect(ends[0].status).toEqual({ code: 'error', message: 'bad' }); + }); + + it('uses _OTHER as error.type when a non-Error is thrown', async () => { + const { exporter, ends } = createSpyExporter(); + await expect( + withSpan(exporter, 's', {}, async () => { + throw 'plain string'; + }) + ).rejects.toBe('plain string'); + expect(ends[0].attributes['error.type']).toBe('_OTHER'); + expect(ends[0].status).toEqual({ code: 'error', message: 'plain string' }); + }); +}); diff --git a/src/execution/tracing.ts b/src/execution/tracing.ts index ac5da6a32..06b5c05ad 100644 --- a/src/execution/tracing.ts +++ b/src/execution/tracing.ts @@ -11,6 +11,16 @@ */ import { nanoid } from 'nanoid'; +import { ErrorAttr, LegacyAttr } from './semconv'; + +/** OpenTelemetry span kind, for the kinds this SDK emits. */ +export type SpanKind = 'internal' | 'client'; + +/** Span outcome; maps to OpenTelemetry's span status. */ +export interface SpanStatus { + code: 'ok' | 'error'; + message?: string; +} /** * A single span of work. @@ -23,6 +33,23 @@ export interface Span { startTime: number; endTime?: number; parentId?: string; + /** Defaults to `internal` when omitted. */ + kind?: SpanKind; + /** Set to `{ code: 'error' }` when the span's work failed. */ + status?: SpanStatus; +} + +/** + * Records a failure on `span`: `error.type` (a low-cardinality class, the + * error's name), the legacy `error` message attribute, and an error status. + */ +export function recordSpanError(span: Span, error: unknown): void { + span.attributes = { + ...span.attributes, + [LegacyAttr.ERROR]: error instanceof Error ? error.message : error, + [ErrorAttr.TYPE]: error instanceof Error ? error.name : ErrorAttr.OTHER, + }; + span.status = { code: 'error', message: error instanceof Error ? error.message : String(error) }; } /** @@ -45,19 +72,22 @@ export interface TraceExporter { * On error, an `error` attribute (the error's message, or the error * itself when it's not an Error) is added to the span's attributes * before `onSpanEnd` is invoked, and the original error is then - * rethrown unchanged. + * rethrown unchanged. The span also gets `error.type` and an error + * `status` (see {@link recordSpanError}). * - `parentId` lets callers thread parent/child span relationships * (LOU-E5 uses this to nest `llm.generate`/`tool.call` spans under a * top-level `agent.run` span). * - `fn` receives the span so callers can read its generated `id` (e.g. * to pass as the `parentId` of a further-nested withSpan() call). + * - `kind` is the OpenTelemetry span kind (default `internal`). */ export async function withSpan( exporter: TraceExporter | undefined, name: string, attributes: Record, fn: (span: Span) => Promise, - parentId?: string + parentId?: string, + kind?: SpanKind ): Promise { const span: Span = { id: nanoid(), @@ -65,6 +95,7 @@ export async function withSpan( attributes, startTime: Date.now(), parentId, + ...(kind ? { kind } : {}), }; exporter?.onSpanStart(span); @@ -72,10 +103,7 @@ export async function withSpan( try { return await fn(span); } catch (error) { - span.attributes = { - ...span.attributes, - error: error instanceof Error ? error.message : error, - }; + recordSpanError(span, error); throw error; } finally { span.endTime = Date.now(); diff --git a/src/flows/FlowExecutor.ts b/src/flows/FlowExecutor.ts index 6c7bf50a3..df7f5122a 100644 --- a/src/flows/FlowExecutor.ts +++ b/src/flows/FlowExecutor.ts @@ -10,6 +10,16 @@ import { AgentConfig } from '../types'; import { SandboxAdapter, NoopSandbox } from '../security/sandboxCore'; import { executeToolWithSandboxGuard } from '../execution/sandboxGuard'; import { evaluateSafeExpression, ExpressionError } from './safeExpression'; +import { Span, TraceExporter, recordSpanError, withSpan } from '../execution/tracing'; +import { + defined, + llmSpanInit, + recordLlmResult, + recordToolOutcome, + resolveCaptureContent, + toolSpanInit, +} from '../execution/genAiSpans'; +import { FLOW_NODE_SPAN_NAME, FlowAttr, GenAiAttr, GenAiOperation } from '../execution/semconv'; /** * Flow execution context @@ -33,6 +43,26 @@ export interface FlowExecutionContext { * so existing callers see no behavior change. */ sandbox?: SandboxAdapter; + /** + * Trace exporter (LOU-D9). When provided, the run is wrapped in an + * `invoke_workflow {flow name}` span, every node execution in a + * `flow.node {type}` span, and each `llmCall`/`toolCall` node's model/tool + * call in a `chat {model}` / `execute_tool {tool}` span (OpenTelemetry + * GenAI conventions, see `semconv.ts`). + */ + exporter?: TraceExporter; + /** + * Id of the span this flow run should nest under, e.g. an agent's + * `invoke_agent` span (`withSpan` hands the span to its callback). + */ + parentSpanId?: string; + /** + * Record prompt/tool-argument content on the `gen_ai.*` attributes. Off by + * default; see `ExecuteOptions.captureContent`. + */ + captureContent?: boolean; + /** Omit the deprecated `prompt`/`args`/`result` attributes. */ + redactContent?: boolean; } /** @@ -110,6 +140,37 @@ export class FlowExecutor { flow: AgentFlow, context: FlowExecutionContext, onEvent?: (event: FlowExecutionEvent) => void + ): Promise { + return withSpan( + context.exporter, + `${GenAiOperation.INVOKE_WORKFLOW} ${flow.name}`, + defined({ + [GenAiAttr.OPERATION_NAME]: GenAiOperation.INVOKE_WORKFLOW, + [GenAiAttr.WORKFLOW_NAME]: flow.name, + [FlowAttr.CODE]: flow.code, + }), + async (flowSpan) => { + const result = await this.runFlow(flow, { ...context, parentSpanId: flowSpan.id }, onEvent); + this.recordOutcome(flowSpan, result.error); + return result; + }, + context.parentSpanId, + 'internal' + ); + } + + /** Records a flow/node outcome (and the error, when it failed) on its span. */ + private static recordOutcome(span: Span, error?: unknown): void { + span.attributes = { ...span.attributes, [FlowAttr.OUTCOME]: error ? 'error' : 'success' }; + if (error) { + recordSpanError(span, error); + } + } + + private static async runFlow( + flow: AgentFlow, + context: FlowExecutionContext, + onEvent?: (event: FlowExecutionEvent) => void ): Promise { const events: FlowExecutionEvent[] = []; const variables = { ...context.variables }; @@ -234,6 +295,34 @@ export class FlowExecutor { const stepId = (node as any).id || `step-${Date.now()}`; + return withSpan( + context.exporter, + `${FLOW_NODE_SPAN_NAME} ${node.type}`, + defined({ [FlowAttr.NODE_ID]: stepId, [FlowAttr.NODE_TYPE]: node.type }), + async (nodeSpan) => { + try { + const result = await this.runNode(node, stepId, { ...context, parentSpanId: nodeSpan.id }, events, onEvent); + this.recordOutcome(nodeSpan); + return result; + } catch (error) { + this.recordOutcome(nodeSpan, error); + throw error; + } + }, + context.parentSpanId, + 'internal' + ); + } + + /** The step-start/step-complete/step-error event bracket around a node's handler. */ + private static async runNode( + node: { type: string }, + stepId: string, + context: FlowExecutionContext, + events: FlowExecutionEvent[], + onEvent?: (event: FlowExecutionEvent) => void + ): Promise { + // Emit step start event emitEvent(events, onEvent, { type: 'step-start', @@ -511,13 +600,25 @@ export class FlowExecutor { data: { model, prompt }, }); - // Call LLM - const result = await context.provider.generate({ - model, - messages, - temperature: node.temperature, - maxTokens: node.maxTokens, + // Call LLM, in a `chat {model}` span under this node's span + const request = { model, messages, temperature: node.temperature, maxTokens: node.maxTokens }; + const captureContent = resolveCaptureContent(context.captureContent); + const init = llmSpanInit(context.provider, request, { + redactContent: context.redactContent, + captureContent, }); + const result = await withSpan( + context.exporter, + init.name, + init.attributes, + async (llmSpan) => { + const generated = await context.provider.generate(request); + recordLlmResult(llmSpan, generated, captureContent); + return generated; + }, + context.parentSpanId, + init.kind + ); // Emit LLM response event emitEvent(events, onEvent, { @@ -558,6 +659,42 @@ export class FlowExecutor { return { toolName, toolDesc }; } + /** + * Run the tool in an `execute_tool {tool}` span under the node's span + */ + private static executeToolInSpan( + toolName: string, + toolDesc: NonNullable>, + args: Record, + sandbox: SandboxAdapter, + context: FlowExecutionContext + ): Promise { + const init = toolSpanInit( + { name: toolName }, + { agent: context.agent, toolRegistry: context.toolRegistry } + ); + return withSpan( + context.exporter, + init.name, + init.attributes, + async (toolSpan) => { + const start = Date.now(); + const result = await executeToolWithSandboxGuard(toolName, toolDesc, args, sandbox); + recordToolOutcome( + toolSpan, + { args, result, latencyMs: Date.now() - start }, + { + redactContent: context.redactContent, + captureContent: resolveCaptureContent(context.captureContent), + } + ); + return result; + }, + context.parentSpanId, + init.kind + ); + } + /** * Execute tool call node */ @@ -586,7 +723,7 @@ export class FlowExecutor { // (LOU-F fix), so this entry point can't silently bypass the sandbox // seam the way it previously did. const sandbox = context.sandbox ?? NoopSandbox; - const result = await executeToolWithSandboxGuard(toolName, toolDesc, args, sandbox); + const result = await this.executeToolInSpan(toolName, toolDesc, args, sandbox, context); // Emit tool result event emitEvent(events, onEvent, {