From 8aa95fd2028367b748b9d857051468f9f46e98e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 15:04:20 +0000 Subject: [PATCH] [EPIC-D][LOU-D48] OpenTelemetry GenAI metrics and a loushy.cost_usd span attribute createOtelTraceExporter() now records gen_ai.client.token.usage and gen_ai.client.operation.duration through @opentelemetry/api's metrics API (global MeterProvider, or the new `meter` option; `metrics: false` disables). Spans gain loushy.cost_usd (per step on chat spans, cumulative on parents via a withSpan() rollup, absent when the model is unpriced) and loushy.usage.estimated on run spans. Adds @opentelemetry/sdk-metrics as a devDependency for tests and the example. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016bY29ibQgv3MDdjAbUT4YR --- CHANGELOG.md | 1 + docs/observability.md | 45 ++++++- examples/tracing/run-otel.ts | 19 +++ examples/tracing/weather-scenario.ts | 2 + llms-full.txt | 45 ++++++- package-lock.json | 18 +++ package.json | 1 + src/execution/genAiSemconv.test.ts | 48 +++++++ src/execution/genAiSpans.ts | 8 +- src/execution/otel.test.ts | 188 ++++++++++++++++++++++++++- src/execution/otel.ts | 88 +++++++++++++ src/execution/semconv.ts | 34 ++++- src/execution/tracing.test.ts | 45 +++++++ src/execution/tracing.ts | 53 +++++++- 14 files changed, 585 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7ce7a09..b72603ceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - 2026-09-28 ### Added +- OpenTelemetry GenAI metrics and cost (LOU-D48): `createOtelTraceExporter()` (`@loushy/build-ai-agent/otel`) now also records the GenAI client metrics `gen_ai.client.token.usage` (histogram, `{token}`, one record per `gen_ai.token.type` `input`/`output`) and `gen_ai.client.operation.duration` (histogram, `s`, for every model call and tool call) through `@opentelemetry/api`'s metrics API: the global `MeterProvider`, or the new `meter` option; `metrics: false` turns them off. Without a registered `MeterProvider` they are discarded by OpenTelemetry's no-op meter. Spans gain `loushy.cost_usd` (estimated USD: per step on `chat` spans, cumulative on `invoke_agent` spans, absent when the price table does not know a model) and `loushy.usage.estimated` on `invoke_agent` spans (it was already on `chat` spans). Any `TraceExporter` sees the cumulative attributes: `withSpan()` sums the cost of finished child spans onto their parent. See docs/observability.md. - Durable sessions (LOU-W9): `agent.session({ id, store, checkpointStore })` (or a `store` that carries both, `{ sessions, checkpoints }`, such as a `SqliteStore`) runs each turn with `sessionId: '.turn-'` and that `CheckpointStore`, so it is checkpointed after every model response and tool result. `session.resume()` finishes a turn interrupted by a crash, a failed checkpoint write or a `PropagatingToolError` (also in a new process) without re-running recorded tool calls or model responses, adds it to the transcript as `send()` would and returns its result (`null` when nothing is pending); `session.pending()` reports an unfinished turn and `session.discardPending()` drops it. `send()` / `stream()` resume a pending turn before sending the new message. A turn paused for approval stays in its checkpoint until it finishes: `resume()` / `send()` throw `SessionAwaitingApprovalError`, and `agent.approvals.resolve()` continues it in the session. Sessions without a checkpoint store are unchanged. See docs/sessions.md. - Structured output (LOU-V4): `createAgent({ output: zodSchema })` (and `ExecuteOptions.output`) makes the final reply a JSON object validated by the schema. `send()`, `run.result` and `agent.approvals.resolve()` resolve with it as `result.object`, typed `z.output` from `send()` and `stream()` of `createAgent()` agents (`SimpleAgent`, `AgentRun` and `ExecutionResult` gain a type parameter that defaults to `unknown`); `result.text` keeps the raw JSON. The system prompt gets an `## Output format` section with the schema as JSON Schema, and each model call carries the new `GenerateOptions.responseFormat: { type: 'json', schema? }` hint, which the `ai`-SDK providers map to `experimental_output` JSON mode. The reply is parsed (a code fence is tolerated) and validated; when invalid, one repair step sends the model the issues (it counts against `maxSteps`), and a reply still invalid ends the run with the new `finishReason: 'output-invalid'`, no `object` and `outputError: { message, issues }`. `run.done` gains an optional `object` field (additive: the event schema version stays 1). See docs/structured-output.md. - Background sub-agents (LOU-Y4): the `task` tool takes `background: true`, which starts the sub-agent and returns `{ taskId, status: 'running', agent }` at once. Agents with `subagents` also get `agent_status({ taskId? })` (`queued`/`running`/`done`/`failed`/`cancelled`/`awaiting-approval`, plus `elapsedMs`), `agent_await({ taskId | taskIds, timeoutMs? })` (the synchronous `task` result, or `status: 'timeout'`) and `agent_cancel({ taskId })`. `withSubagentOptions(subagents, { maxConcurrent })` bounds how many run at once per lead run (default 3; the rest queue). Background sub-agents inherit the lead run's runtime and `maxSubagentDepth`, and aborting the lead cancels them. Not yet (LOU-Y4.2): cancelling or awaiting them when the lead run finishes, and resuming one that paused for approval (it reports `awaiting-approval` with its `approvalId`). A user tool named `agent_status`, `agent_await` or `agent_cancel` now conflicts with `subagents`, like `task`. See docs/sub-agents.md. diff --git a/docs/observability.md b/docs/observability.md index 2f49a091a..e6fbe797b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -31,7 +31,7 @@ For real OpenTelemetry spans, import `createOtelTraceExporter()` from `@loushy/build-ai-agent/otel` (needs the optional peer dependency `@opentelemetry/api` and a registered `TracerProvider`). It carries span kind and error status over to OpenTelemetry. See `examples/tracing` -(`npm run example:tracing:console`, `npm run example:tracing:otel`). +(`npm run example:tracing:console`, `npm run example:tracing:otel`, which also prints metrics). ## Span tree @@ -57,7 +57,7 @@ child. ## Attributes Constants for every name live in `src/execution/semconv.ts` (exported as -`GenAiAttr`, `GenAiOperation`, `ErrorAttr`, `FlowAttr`, `LegacyAttr`). +`GenAiAttr`, `GenAiOperation`, `GenAiMetric`, `ErrorAttr`, `FlowAttr`, `SdkAttr`, `LegacyAttr`). ### Agent run: `invoke_agent {agent name}` @@ -68,6 +68,8 @@ Constants for every name live in `src/execution/semconv.ts` (exported as | `gen_ai.agent.id` | The agent's `id`, when set | | `gen_ai.provider.name` | The provider's `name` | | `gen_ai.conversation.id` | `sessionId`, when set | +| `loushy.cost_usd` | Cumulative estimated USD of the run (every model call and delegated child run); absent when any model used has no known price | +| `loushy.usage.estimated` | `true` when any of the run's tokens were estimated | ### Model call: `chat {model}` (CLIENT) @@ -80,6 +82,8 @@ Constants for every name live in `src/execution/semconv.ts` (exported as | `gen_ai.response.model` | When the provider reports it | | `gen_ai.response.finish_reasons` | e.g. `["stop"]`, `["tool_call"]` | | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | Token usage | +| `loushy.cost_usd` | Estimated USD of this step, when the built-in price table (or `registerModel`) knows the model; absent otherwise | +| `loushy.usage.estimated` | `true` when the provider reported no usage and the tokens were estimated | ### Tool call: `execute_tool {tool}` @@ -114,6 +118,43 @@ A failed span gets `error.type` and an error span status (OpenTelemetry `name` (`_OTHER` when a non-`Error` is thrown). A tool call that returns an error to the model (`isError`) is marked `error.type = tool_error`. +## Metrics + +`createOtelTraceExporter()` also records the OpenTelemetry GenAI client +metrics through `@opentelemetry/api`'s metrics API, using the global +`MeterProvider` (`metrics.getMeter(...)`), or the `meter` option: + +| Metric | Type, unit | Recorded | Attributes | +| --- | --- | --- | --- | +| `gen_ai.client.token.usage` | Histogram, `{token}` | Once per token type for every model call that reports (or estimates) usage | `gen_ai.operation.name` (`chat`), `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model` (when known), `gen_ai.token.type` (`input` or `output`) | +| `gen_ai.client.operation.duration` | Histogram, `s` | Every model call (`chat`) and tool call (`execute_tool`) | `gen_ai.operation.name`, `gen_ai.provider.name` and `gen_ai.request.model` (model calls), `gen_ai.response.model` (when known), `error.type` (when it failed) | + +Both use the spec's recommended bucket boundaries. They follow the +`gen-ai-metrics` page of semantic-conventions v1.40 (`Development` stability): +the newest conventions split the token histogram into per-type counters, +which this SDK does not emit yet. Agent and flow spans are not counted in the +duration metric, only the model and tool calls inside them. + +```ts +import { metrics } from '@opentelemetry/api'; +import { ConsoleMetricExporter, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { createOtelTraceExporter } from '@loushy/build-ai-agent/otel'; + +// Swap ConsoleMetricExporter for an OTLP exporter to reach any backend. +metrics.setGlobalMeterProvider( + new MeterProvider({ readers: [new PeriodicExportingMetricReader({ exporter: new ConsoleMetricExporter() })] }) +); + +const exporter = createOtelTraceExporter(); // metrics on by default +// createOtelTraceExporter({ metrics: false }) // spans only +// createOtelTraceExporter({ meter }) // an already-obtained Meter +``` + +Without a registered `MeterProvider`, OpenTelemetry's no-op meter discards +the records. The metrics API lives in `@opentelemetry/api`, so nothing is +recorded (and nothing is loaded) when that optional peer is not installed. +The cost is an estimate from the price table (see `registerModel`), not a bill. + ## Message and argument content is opt-in Prompts, model output and tool arguments/results are sensitive and large, so diff --git a/examples/tracing/run-otel.ts b/examples/tracing/run-otel.ts index ab492fdcf..fbb2f76df 100644 --- a/examples/tracing/run-otel.ts +++ b/examples/tracing/run-otel.ts @@ -9,10 +9,21 @@ * that decides *where* the spans go; the SDK-to-OTel translation is no * longer hand-rolled here. * + * LOU-D48: the same exporter also records the GenAI metrics + * (`gen_ai.client.token.usage`, `gen_ai.client.operation.duration`) through + * whatever global MeterProvider is registered; here an OTel MeterProvider + * prints them to the console on shutdown. The spans carry `loushy.cost_usd`. + * * Run with: * npm run example:tracing:otel */ +import { metrics } from '@opentelemetry/api'; +import { + ConsoleMetricExporter, + MeterProvider, + PeriodicExportingMetricReader, +} from '@opentelemetry/sdk-metrics'; import { NodeTracerProvider, ConsoleSpanExporter, @@ -28,6 +39,13 @@ async function main() { provider.register(); const tracer = provider.getTracer('loushy-tracing-example'); + // Metrics go wherever the global MeterProvider sends them. `shutdown()` + // below flushes them to the console once. + const meterProvider = new MeterProvider({ + readers: [new PeriodicExportingMetricReader({ exporter: new ConsoleMetricExporter() })], + }); + metrics.setGlobalMeterProvider(meterProvider); + // In real usage the tracer name is enough - `createOtelTraceExporter()` // will resolve one via `trace.getTracer(...)` for you. Passing the // tracer explicitly here just reuses the one obtained above. @@ -36,6 +54,7 @@ async function main() { await runWeatherScenario(exporter); await provider.shutdown(); + await meterProvider.shutdown(); } main().catch((error) => { diff --git a/examples/tracing/weather-scenario.ts b/examples/tracing/weather-scenario.ts index 87486d10f..edc70256b 100644 --- a/examples/tracing/weather-scenario.ts +++ b/examples/tracing/weather-scenario.ts @@ -77,6 +77,8 @@ function createWeatherAgent() { .setType(AgentType.SmartAssistant) .setName('Weather Agent') .setPrompt('You are a helpful weather assistant.') + // A priced model id, so spans carry `loushy.cost_usd` (the provider is still a mock). + .setSettings({ model: 'gpt-4o-mini' }) .addTool('getWeather', { tool: 'getWeather', options: {} }) .build(); } diff --git a/llms-full.txt b/llms-full.txt index d046fd03e..eb54c1603 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -3356,7 +3356,7 @@ For real OpenTelemetry spans, import `createOtelTraceExporter()` from `@loushy/build-ai-agent/otel` (needs the optional peer dependency `@opentelemetry/api` and a registered `TracerProvider`). It carries span kind and error status over to OpenTelemetry. See `examples/tracing` -(`npm run example:tracing:console`, `npm run example:tracing:otel`). +(`npm run example:tracing:console`, `npm run example:tracing:otel`, which also prints metrics). ## Span tree @@ -3382,7 +3382,7 @@ child. ## Attributes Constants for every name live in `src/execution/semconv.ts` (exported as -`GenAiAttr`, `GenAiOperation`, `ErrorAttr`, `FlowAttr`, `LegacyAttr`). +`GenAiAttr`, `GenAiOperation`, `GenAiMetric`, `ErrorAttr`, `FlowAttr`, `SdkAttr`, `LegacyAttr`). ### Agent run: `invoke_agent {agent name}` @@ -3393,6 +3393,8 @@ Constants for every name live in `src/execution/semconv.ts` (exported as | `gen_ai.agent.id` | The agent's `id`, when set | | `gen_ai.provider.name` | The provider's `name` | | `gen_ai.conversation.id` | `sessionId`, when set | +| `loushy.cost_usd` | Cumulative estimated USD of the run (every model call and delegated child run); absent when any model used has no known price | +| `loushy.usage.estimated` | `true` when any of the run's tokens were estimated | ### Model call: `chat {model}` (CLIENT) @@ -3405,6 +3407,8 @@ Constants for every name live in `src/execution/semconv.ts` (exported as | `gen_ai.response.model` | When the provider reports it | | `gen_ai.response.finish_reasons` | e.g. `["stop"]`, `["tool_call"]` | | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | Token usage | +| `loushy.cost_usd` | Estimated USD of this step, when the built-in price table (or `registerModel`) knows the model; absent otherwise | +| `loushy.usage.estimated` | `true` when the provider reported no usage and the tokens were estimated | ### Tool call: `execute_tool {tool}` @@ -3439,6 +3443,43 @@ A failed span gets `error.type` and an error span status (OpenTelemetry `name` (`_OTHER` when a non-`Error` is thrown). A tool call that returns an error to the model (`isError`) is marked `error.type = tool_error`. +## Metrics + +`createOtelTraceExporter()` also records the OpenTelemetry GenAI client +metrics through `@opentelemetry/api`'s metrics API, using the global +`MeterProvider` (`metrics.getMeter(...)`), or the `meter` option: + +| Metric | Type, unit | Recorded | Attributes | +| --- | --- | --- | --- | +| `gen_ai.client.token.usage` | Histogram, `{token}` | Once per token type for every model call that reports (or estimates) usage | `gen_ai.operation.name` (`chat`), `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model` (when known), `gen_ai.token.type` (`input` or `output`) | +| `gen_ai.client.operation.duration` | Histogram, `s` | Every model call (`chat`) and tool call (`execute_tool`) | `gen_ai.operation.name`, `gen_ai.provider.name` and `gen_ai.request.model` (model calls), `gen_ai.response.model` (when known), `error.type` (when it failed) | + +Both use the spec's recommended bucket boundaries. They follow the +`gen-ai-metrics` page of semantic-conventions v1.40 (`Development` stability): +the newest conventions split the token histogram into per-type counters, +which this SDK does not emit yet. Agent and flow spans are not counted in the +duration metric, only the model and tool calls inside them. + +```ts +import { metrics } from '@opentelemetry/api'; +import { ConsoleMetricExporter, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { createOtelTraceExporter } from '@loushy/build-ai-agent/otel'; + +// Swap ConsoleMetricExporter for an OTLP exporter to reach any backend. +metrics.setGlobalMeterProvider( + new MeterProvider({ readers: [new PeriodicExportingMetricReader({ exporter: new ConsoleMetricExporter() })] }) +); + +const exporter = createOtelTraceExporter(); // metrics on by default +// createOtelTraceExporter({ metrics: false }) // spans only +// createOtelTraceExporter({ meter }) // an already-obtained Meter +``` + +Without a registered `MeterProvider`, OpenTelemetry's no-op meter discards +the records. The metrics API lives in `@opentelemetry/api`, so nothing is +recorded (and nothing is loaded) when that optional peer is not installed. +The cost is an estimate from the price table (see `registerModel`), not a bill. + ## Message and argument content is opt-in Prompts, model output and tool arguments/results are sensitive and large, so diff --git a/package-lock.json b/package-lock.json index f24a53999..eab085217 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@ai-sdk/openai": "^0.0.42", "@modelcontextprotocol/sdk": "^1.30.1", "@opentelemetry/api": "^1.9.1", + "@opentelemetry/sdk-metrics": "^2.11.0", "@opentelemetry/sdk-trace-node": "^2.11.0", "@types/dockerode": "^4.0.1", "@types/node": "^20", @@ -2885,6 +2886,23 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.11.0.tgz", + "integrity": "sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/sdk-trace-base": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz", diff --git a/package.json b/package.json index 3a101ecd0..594afa9b0 100644 --- a/package.json +++ b/package.json @@ -205,6 +205,7 @@ "@ai-sdk/openai": "^0.0.42", "@modelcontextprotocol/sdk": "^1.30.1", "@opentelemetry/api": "^1.9.1", + "@opentelemetry/sdk-metrics": "^2.11.0", "@opentelemetry/sdk-trace-node": "^2.11.0", "@types/dockerode": "^4.0.1", "@types/node": "^20", diff --git a/src/execution/genAiSemconv.test.ts b/src/execution/genAiSemconv.test.ts index 8d4a5b615..a1d683e22 100644 --- a/src/execution/genAiSemconv.test.ts +++ b/src/execution/genAiSemconv.test.ts @@ -388,3 +388,51 @@ describe('traced flows (LOU-D9)', () => { expect(on.spans[3].attributes['gen_ai.tool.call.arguments']).toBe(JSON.stringify({ city: 'Paris' })); }); }); + +describe('loushy.cost_usd and loushy.usage.estimated spans (LOU-D48)', () => { + async function runPriced(model: string, scripted: boolean) { + const { agent, toolRegistry } = setup(); + const { exporter, spans } = memoryExporter(); + const provider = scripted + ? modelWithOneToolCall() + : mockModel([{ toolCalls: [{ id: 'c1', name: 'get_weather', args: { city: 'Paris' } }] }, 'It is 21C.']); + const result = await AgentExecutor.execute({ + agent: { ...agent, settings: { model } }, + input: 'Weather in Paris?', + provider, + toolRegistry, + exporter, + }); + return { spans, result }; + } + + const gpt4oMini = (input: number, output: number) => (input * 0.15 + output * 0.6) / 1e6; + + it('records the step cost on each chat span and the cumulative cost on the invoke_agent span', async () => { + const { spans, result } = await runPriced('gpt-4o-mini', true); + const [run, chat1, tool, chat2] = spans; + + expect(chat1.attributes['loushy.cost_usd']).toBeCloseTo(gpt4oMini(10, 5), 12); + expect(chat2.attributes['loushy.cost_usd']).toBeCloseTo(gpt4oMini(20, 7), 12); + expect(run.attributes['loushy.cost_usd']).toBeCloseTo(gpt4oMini(30, 12), 12); + expect(run.attributes['loushy.cost_usd']).toBeCloseTo(result.usage.costUsd as number, 12); + expect(tool.attributes).not.toHaveProperty('loushy.cost_usd'); + for (const span of [run, chat1, chat2]) expect(span.attributes).not.toHaveProperty('loushy.usage.estimated'); + }); + + it('omits loushy.cost_usd when the model has no known price', async () => { + const { spans } = await runPriced('test-model', true); + expect(spans.some((s) => 'loushy.cost_usd' in s.attributes)).toBe(false); + }); + + it('flags loushy.usage.estimated on the chat spans and the run span when tokens were estimated', async () => { + const { spans, result } = await runPriced('gpt-4o-mini', false); + const [run, chat1, , chat2] = spans; + + expect(result.usage.estimated).toBe(true); + expect(run.attributes['loushy.usage.estimated']).toBe(true); + expect(chat1.attributes['loushy.usage.estimated']).toBe(true); + expect(chat2.attributes['loushy.usage.estimated']).toBe(true); + expect(run.attributes['loushy.cost_usd']).toBeCloseTo(result.usage.costUsd as number, 12); + }); +}); diff --git a/src/execution/genAiSpans.ts b/src/execution/genAiSpans.ts index 8737d2db3..40cd0c9e0 100644 --- a/src/execution/genAiSpans.ts +++ b/src/execution/genAiSpans.ts @@ -17,6 +17,7 @@ import type { ToolRegistry } from '../tools'; import type { AgentConfig } from '../types'; import { ErrorAttr, GenAiAttr, GenAiOperation, LegacyAttr, SdkAttr, TOOL_TYPE_FUNCTION, CAPTURE_CONTENT_ENV } from './semconv'; import type { Span, SpanKind } from './tracing'; +import { estimateCost } from '../models'; import { normalizeUsage } from '../models/usage'; import type { Usage } from '../models/usage'; @@ -164,14 +165,18 @@ function responseModel(generated: GenerateResult): string | undefined { * Records usage, finish reason, response model (and output content) on a `chat` span. * Usage is the same normalized `Usage` the run totals use: pass the executor's * `measured` figures (which may be estimates), else what the provider reported. + * `loushy.cost_usd` is the step's cost when the price table knows the model. */ export function recordLlmResult( span: Span, generated: GenerateResult, captureContent: boolean, - measured?: { usage: Usage; estimated: boolean } + measured?: { usage: Usage; estimated: boolean; costUsd?: number } ): void { const usage = measured?.usage ?? normalizeUsage(generated.usage); + const model = span.attributes[GenAiAttr.REQUEST_MODEL]; + const costUsd = + measured?.costUsd ?? (usage && typeof model === 'string' ? estimateCost(usage, model) : undefined); const parts: unknown[] = generated.text ? [textPart(generated.text)] : []; parts.push(...(generated.toolCalls ?? []).map(toolCallPart)); // Token counts and finish reason are never redacted. @@ -186,6 +191,7 @@ export function recordLlmResult( [LegacyAttr.COMPLETION_TOKENS]: usage?.outputTokens, [LegacyAttr.TOTAL_TOKENS]: usage?.totalTokens, [SdkAttr.USAGE_ESTIMATED]: measured?.estimated ? true : undefined, + [SdkAttr.COST_USD]: costUsd, [LegacyAttr.FINISH_REASON]: generated.finishReason, [GenAiAttr.OUTPUT_MESSAGES]: captureContent ? JSON.stringify([{ role: 'assistant', parts }]) diff --git a/src/execution/otel.test.ts b/src/execution/otel.test.ts index 2368ee5d1..46ea634e8 100644 --- a/src/execution/otel.test.ts +++ b/src/execution/otel.test.ts @@ -1,8 +1,18 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import type { Tracer, Span as OtelSpan, Context } from '@opentelemetry/api'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { readdirSync, readFileSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { z } from 'zod'; +import * as otelApi from '@opentelemetry/api'; +import type { Tracer, Span as OtelSpan, Context, Meter } from '@opentelemetry/api'; import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; +import { AggregationTemporality, InMemoryMetricExporter, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; import { createOtelTraceExporter } from './otel'; -import { Span } from './tracing'; +import { AgentExecutor } from './AgentExecutor'; +import { Span, TraceExporter } from './tracing'; +import { ToolRegistry, defineTool } from '../tools'; +import { AgentBuilder } from '../core'; +import { AgentType } from '../types'; +import { mockModel } from '../testing'; /** * A minimal fake OTel span that just records what was done to it, so @@ -195,3 +205,175 @@ describe('createOtelTraceExporter kind, status and array attributes (LOU-D9)', ( }); }); }); + +describe('createOtelTraceExporter GenAI metrics (LOU-D48)', () => { + interface Recorded { + name: string; + value: number; + attributes: Record; + } + interface Instrument { + name: string; + options: { unit?: string; advice?: { explicitBucketBoundaries?: number[] } }; + } + + /** A minimal recording Meter: every histogram record lands in `records`. */ + function makeRecordingMeter() { + const records: Recorded[] = []; + const instruments: Instrument[] = []; + const meter = { + createHistogram: (name: string, options: Instrument['options']) => { + instruments.push({ name, options }); + return { record: (value: number, attributes: Record = {}) => records.push({ name, value, attributes }) }; + }, + } as unknown as Meter; + return { meter, records, instruments }; + } + + const weather = defineTool({ + name: 'get_weather', + description: 'Get the weather for a city', + input: z.object({ city: z.string() }), + execute: async ({ city }) => ({ city, tempC: 21 }), + }); + + /** One run with a priced model and one tool call, exported through `exporter`. */ + async function runPricedAgent(exporter: TraceExporter) { + const toolRegistry = new ToolRegistry(); + toolRegistry.register(weather); + const agent = { + ...AgentBuilder.create() + .setType(AgentType.SmartAssistant) + .setName('Weather Bot') + .addTool('get_weather', { tool: 'get_weather', options: {} }) + .build(), + id: 'agent-1', + settings: { model: 'gpt-4o-mini' }, + }; + const provider = mockModel([ + { toolCalls: [{ name: 'get_weather', args: { city: 'Paris' } }], usage: { inputTokens: 10, outputTokens: 5 } }, + { text: 'It is 21C.', usage: { inputTokens: 20, outputTokens: 7 } }, + ]); + await AgentExecutor.execute({ agent, input: 'Weather?', provider, toolRegistry, exporter }); + return provider; + } + + it('records token usage and duration histograms for each model call of a run', async () => { + const fake = makeRecordingMeter(); + const provider = await runPricedAgent(createOtelTraceExporter({ tracer: makeFakeTracer().tracer, meter: fake.meter })); + + expect(fake.instruments).toEqual([ + expect.objectContaining({ name: 'gen_ai.client.token.usage', options: expect.objectContaining({ unit: '{token}' }) }), + expect.objectContaining({ name: 'gen_ai.client.operation.duration', options: expect.objectContaining({ unit: 's' }) }), + ]); + expect(fake.instruments[0].options.advice?.explicitBucketBoundaries?.[0]).toBe(1); + expect(fake.instruments[1].options.advice?.explicitBucketBoundaries?.[0]).toBe(0.01); + + const chat = { 'gen_ai.operation.name': 'chat', 'gen_ai.provider.name': provider.name, 'gen_ai.request.model': 'gpt-4o-mini' }; + const tokens = fake.records.filter((r) => r.name === 'gen_ai.client.token.usage'); + expect(tokens.map((r) => [r.value, r.attributes['gen_ai.token.type']])).toEqual([ + [10, 'input'], + [5, 'output'], + [20, 'input'], + [7, 'output'], + ]); + for (const record of tokens) expect(record.attributes).toMatchObject(chat); + + const durations = fake.records.filter((r) => r.name === 'gen_ai.client.operation.duration'); + // two model calls and one tool call, in the order the spans ended + expect(durations.map((r) => r.attributes['gen_ai.operation.name'])).toEqual(['chat', 'execute_tool', 'chat']); + expect(durations[0].attributes).toMatchObject(chat); + expect(durations[1].attributes).toEqual({ 'gen_ai.operation.name': 'execute_tool' }); + for (const record of durations) { + expect(record.value).toBeGreaterThanOrEqual(0); + expect(record.attributes).not.toHaveProperty('gen_ai.token.type'); + } + }); + + it('records gen_ai.response.model and error.type when the spans have them', () => { + const fake = makeRecordingMeter(); + const exporter = createOtelTraceExporter({ tracer: makeFakeTracer().tracer, meter: fake.meter }); + const attributes = { 'gen_ai.operation.name': 'chat', 'gen_ai.response.model': 'gpt-4o-mini-2024', 'error.type': 'Error' }; + exporter.onSpanStart(makeSpan({ id: 'c', attributes })); + exporter.onSpanEnd(makeSpan({ id: 'c', attributes, startTime: 1000, endTime: 3500 })); + + expect(fake.records).toEqual([ + { name: 'gen_ai.client.operation.duration', value: 2.5, attributes }, + ]); + }); + + it('records nothing for non-model, non-tool spans, and no tokens when none were reported', () => { + const fake = makeRecordingMeter(); + const exporter = createOtelTraceExporter({ tracer: makeFakeTracer().tracer, meter: fake.meter }); + exporter.onSpanStart(makeSpan({ id: 'r', attributes: { 'gen_ai.operation.name': 'invoke_agent', 'gen_ai.usage.input_tokens': 3 } })); + exporter.onSpanEnd(makeSpan({ id: 'r', endTime: 2000, attributes: { 'gen_ai.operation.name': 'invoke_agent', 'gen_ai.usage.input_tokens': 3 } })); + exporter.onSpanStart(makeSpan({ id: 'x', attributes: {} })); + exporter.onSpanEnd(makeSpan({ id: 'x', endTime: 2000, attributes: {} })); + exporter.onSpanStart(makeSpan({ id: 'c', attributes: { 'gen_ai.operation.name': 'chat' } })); + exporter.onSpanEnd(makeSpan({ id: 'c', endTime: 2000, attributes: { 'gen_ai.operation.name': 'chat' } })); + + expect(fake.records.map((r) => r.name)).toEqual(['gen_ai.client.operation.duration']); + }); + + it('creates no instruments when metrics is false', async () => { + const fake = makeRecordingMeter(); + await runPricedAgent(createOtelTraceExporter({ tracer: makeFakeTracer().tracer, meter: fake.meter, metrics: false })); + expect(fake.instruments).toEqual([]); + expect(fake.records).toEqual([]); + }); + + describe('with the global MeterProvider', () => { + afterEach(() => { + otelApi.metrics.disable(); + }); + + it('resolves the meter via metrics.getMeter(tracerName, tracerVersion) by default', async () => { + const fake = makeRecordingMeter(); + const getMeter = vi.fn(() => fake.meter); + otelApi.metrics.setGlobalMeterProvider({ getMeter }); + + await runPricedAgent(createOtelTraceExporter({ tracer: makeFakeTracer().tracer, tracerName: 'my-agent', tracerVersion: '1.2.3' })); + + expect(getMeter).toHaveBeenCalledWith('my-agent', '1.2.3', undefined); + expect(fake.records.some((r) => r.name === 'gen_ai.client.token.usage')).toBe(true); + }); + + it('records into a real SDK MeterProvider', async () => { + const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE); + const reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 }); + const meterProvider = new MeterProvider({ readers: [reader] }); + otelApi.metrics.setGlobalMeterProvider(meterProvider); + + await runPricedAgent(createOtelTraceExporter({ tracer: makeFakeTracer().tracer })); + + await reader.forceFlush(); + const collected = exporter.getMetrics().flatMap((rm) => rm.scopeMetrics.flatMap((scope) => scope.metrics)); + const byName = Object.fromEntries(collected.map((m) => [m.descriptor.name, m])); + expect(byName['gen_ai.client.token.usage'].descriptor.unit).toBe('{token}'); + expect(byName['gen_ai.client.operation.duration'].descriptor.unit).toBe('s'); + + const sums = (byName['gen_ai.client.token.usage'].dataPoints as Array<{ attributes: Record; value: { sum?: number; count: number } }>) + .map((p) => [p.attributes['gen_ai.token.type'], p.value.sum, p.value.count]); + expect(sums).toEqual(expect.arrayContaining([['input', 30, 2], ['output', 12, 2]])); + await meterProvider.shutdown(); + }); + }); +}); + +describe('optional peer', () => { + it('only the /otel entry imports @opentelemetry/api, so the main package works without it', () => { + const root = join(__dirname, '..'); + const importers: string[] = []; + const visit = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) visit(path); + else if (/\.tsx?$/.test(entry.name) && !/\.test\.tsx?$/.test(entry.name)) { + if (/from\s+['"]@opentelemetry\//.test(readFileSync(path, 'utf8'))) importers.push(relative(root, path)); + } + } + }; + visit(root); + expect(importers).toEqual([join('execution', 'otel.ts')]); + }); +}); diff --git a/src/execution/otel.ts b/src/execution/otel.ts index 29a8339ca..c676cb87c 100644 --- a/src/execution/otel.ts +++ b/src/execution/otel.ts @@ -25,13 +25,26 @@ import { trace, context, + metrics as otelMetrics, + Attributes, AttributeValue, Context, + Histogram, + Meter, Span as OtelSpan, SpanKind as OtelSpanKind, SpanStatusCode, Tracer, } from '@opentelemetry/api'; +import { + ErrorAttr, + GenAiAttr, + GenAiMetric, + GenAiOperation, + OPERATION_DURATION_BUCKETS, + TOKEN_USAGE_BUCKETS, + TokenType, +} from './semconv'; import { Span, TraceExporter } from './tracing'; /** @@ -55,6 +68,19 @@ export interface OtelTraceExporterOptions { * tracer instance is used. */ tracer?: Tracer; + /** + * Record the OpenTelemetry GenAI client metrics `gen_ai.client.token.usage` + * and `gen_ai.client.operation.duration` (one record per model call, and a + * duration per tool call) alongside the spans (LOU-D48). Default `true`; + * without a registered `MeterProvider` OTel's no-op meter discards them. + * See docs/observability.md. + */ + metrics?: boolean; + /** + * Use an already-obtained OTel `Meter` instead of + * `metrics.getMeter(tracerName, tracerVersion)`. Ignored when `metrics` is `false`. + */ + meter?: Meter; } /** @@ -90,6 +116,8 @@ export function createOtelTraceExporter(options: OtelTraceExporterOptions = {}): // Our Span.id -> the OTel span + context it was started in, so a child // Span (matched by parentId) can be started as a child of the right // OTel context instead of the ambient one. + const recordMetrics = resolveMetricsRecorder(options); + const otelSpansById = new Map(); return { @@ -124,10 +152,70 @@ export function createOtelTraceExporter(options: OtelTraceExporterOptions = {}): } entry.span.end(span.endTime); otelSpansById.delete(span.id); + recordMetrics(span); }, }; } +/** The metrics recorder for `options`: a no-op when `metrics` is `false`. */ +function resolveMetricsRecorder(options: OtelTraceExporterOptions): (span: Span) => void { + if (options.metrics === false) return () => undefined; + const name = options.tracerName ?? '@loushy/build-ai-agent'; + return createMetricsRecorder(options.meter ?? otelMetrics.getMeter(name, options.tracerVersion)); +} + +/** + * Returns the function that records a finished span's GenAI client metrics: + * duration for `chat` and `execute_tool` spans, and token usage (input and + * output) for `chat` spans that carry token counts. + */ +function createMetricsRecorder(meter: Meter): (span: Span) => void { + const tokenUsage = meter.createHistogram(GenAiMetric.TOKEN_USAGE, { + description: 'Number of input and output tokens used', + unit: '{token}', + advice: { explicitBucketBoundaries: TOKEN_USAGE_BUCKETS }, + }); + const duration = meter.createHistogram(GenAiMetric.OPERATION_DURATION, { + description: 'GenAI operation duration', + unit: 's', + advice: { explicitBucketBoundaries: OPERATION_DURATION_BUCKETS }, + }); + + return (span) => { + const attributes = metricAttributes(span); + if (!attributes) return; + if (span.endTime !== undefined) { + duration.record((span.endTime - span.startTime) / 1000, attributes); + } + recordTokens(tokenUsage, span, attributes); + }; +} + +/** Metric attributes of a `chat` / `execute_tool` span; `undefined` for any other span. */ +function metricAttributes(span: Span): Attributes | undefined { + const operation = span.attributes[GenAiAttr.OPERATION_NAME]; + if (operation !== GenAiOperation.CHAT && operation !== GenAiOperation.EXECUTE_TOOL) { + return undefined; + } + const keys = [GenAiAttr.PROVIDER_NAME, GenAiAttr.REQUEST_MODEL, GenAiAttr.RESPONSE_MODEL, ErrorAttr.TYPE]; + const attributes: Attributes = { [GenAiAttr.OPERATION_NAME]: operation }; + for (const key of keys) { + const value = span.attributes[key]; + if (typeof value === 'string') attributes[key] = value; + } + return attributes; +} + +function recordTokens(histogram: Histogram, span: Span, attributes: Attributes): void { + const counts = { + [TokenType.INPUT]: span.attributes[GenAiAttr.USAGE_INPUT_TOKENS], + [TokenType.OUTPUT]: span.attributes[GenAiAttr.USAGE_OUTPUT_TOKENS], + }; + for (const [type, count] of Object.entries(counts)) { + if (typeof count === 'number') histogram.record(count, { ...attributes, [GenAiAttr.TOKEN_TYPE]: type }); + } +} + function setAttributes(otelSpan: OtelSpan, attributes: Record): void { for (const [key, value] of Object.entries(attributes)) { if (value !== undefined) { diff --git a/src/execution/semconv.ts b/src/execution/semconv.ts index ef42b8b5e..00324f3c6 100644 --- a/src/execution/semconv.ts +++ b/src/execution/semconv.ts @@ -54,8 +54,34 @@ export const GenAiAttr = { OUTPUT_MESSAGES: 'gen_ai.output.messages', TOOL_CALL_ARGUMENTS: 'gen_ai.tool.call.arguments', TOOL_CALL_RESULT: 'gen_ai.tool.call.result', + /** Metric attribute: `input` or `output` (see {@link TokenType}). */ + TOKEN_TYPE: 'gen_ai.token.type', } as const; +/** `gen_ai.token.type` values of the `gen_ai.client.token.usage` metric. */ +export const TokenType = { INPUT: 'input', OUTPUT: 'output' } as const; + +/** + * OpenTelemetry GenAI client metrics (LOU-D48). Spec: gen-ai-metrics.md of + * semantic-conventions v1.40 (the newer semantic-conventions-genai repo + * replaces the token histogram with per-type counters, which are not + * emitted here). Both are `Development` stability; the bucket boundaries are + * the spec's recommendations. + */ +export const GenAiMetric = { + /** Histogram, unit `{token}`, one record per token type per model call. */ + TOKEN_USAGE: 'gen_ai.client.token.usage', + /** Histogram, unit `s`, one record per model call and per tool call. */ + OPERATION_DURATION: 'gen_ai.client.operation.duration', +} as const; + +export const TOKEN_USAGE_BUCKETS = [ + 1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, +]; +export const OPERATION_DURATION_BUCKETS = [ + 0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92, +]; + /** `error.type` (stable in semconv) and its `_OTHER` fallback. */ export const ErrorAttr = { TYPE: 'error.type', @@ -68,8 +94,14 @@ export const TOOL_TYPE_FUNCTION = 'function'; /** Attributes specific to this SDK (not part of the GenAI semantic conventions). */ export const SdkAttr = { - /** `true` on a `chat` span whose token counts are estimates because the provider reported none (LOU-V5). */ + /** `true` on a span whose token counts are (partly) estimates because the provider reported none (LOU-V5). */ USAGE_ESTIMATED: 'loushy.usage.estimated', + /** + * Estimated USD (LOU-D48): per step on a `chat` span; cumulative on the + * parent spans (`invoke_agent`, and a delegating `execute_tool`). Absent + * when the price table does not know a contributing model. + */ + COST_USD: 'loushy.cost_usd', } as const; /** diff --git a/src/execution/tracing.test.ts b/src/execution/tracing.test.ts index 5e992f49c..28a1faae8 100644 --- a/src/execution/tracing.test.ts +++ b/src/execution/tracing.test.ts @@ -107,3 +107,48 @@ describe('withSpan kind and error status (LOU-D9)', () => { expect(ends[0].status).toEqual({ code: 'error', message: 'plain string' }); }); }); + +describe('withSpan cost rollup (LOU-D48)', () => { + /** Runs `children` (their attributes) as child spans of one parent span and returns the parent's final attributes. */ + async function parentAttributes(children: Array>) { + const { exporter, ends } = createSpyExporter(); + await withSpan(exporter, 'parent', {}, async (parent) => { + for (const attrs of children) await withSpan(exporter, 'child', attrs, async () => undefined, parent.id); + }); + return ends[ends.length - 1].attributes; + } + + it('sums the loushy.cost_usd of finished child spans onto the parent', async () => { + const attrs = await parentAttributes([{ 'loushy.cost_usd': 0.25 }, { 'loushy.cost_usd': 0.5 }]); + expect(attrs['loushy.cost_usd']).toBe(0.75); + }); + + it('reports a zero cost for priced children that cost nothing', async () => { + expect((await parentAttributes([{ 'loushy.cost_usd': 0 }]))['loushy.cost_usd']).toBe(0); + }); + + it('omits the sum when a child with token usage has no price', async () => { + const attrs = await parentAttributes([{ 'loushy.cost_usd': 0.25 }, { 'gen_ai.usage.input_tokens': 5 }]); + expect(attrs).not.toHaveProperty('loushy.cost_usd'); + }); + + it('adds nothing for children without usage, and propagates through nested spans', async () => { + expect(await parentAttributes([{ other: 1 }])).toEqual({}); + + const { exporter, ends } = createSpyExporter(); + await withSpan(exporter, 'run', {}, async (run) => { + await withSpan( + exporter, + 'sub-run', + {}, + async (sub) => { + const chatAttrs = { 'loushy.cost_usd': 1, 'loushy.usage.estimated': true }; + await withSpan(exporter, 'chat', chatAttrs, async () => undefined, sub.id); + }, + run.id + ); + }); + const run = ends[ends.length - 1]; + expect(run.attributes).toEqual({ 'loushy.cost_usd': 1, 'loushy.usage.estimated': true }); + }); +}); diff --git a/src/execution/tracing.ts b/src/execution/tracing.ts index 5cdcd7f74..190b05b2e 100644 --- a/src/execution/tracing.ts +++ b/src/execution/tracing.ts @@ -11,7 +11,7 @@ */ import { newId } from '../utils/id'; -import { ErrorAttr, LegacyAttr } from './semconv'; +import { ErrorAttr, GenAiAttr, LegacyAttr, SdkAttr } from './semconv'; /** OpenTelemetry span kind, for the kinds this SDK emits. */ export type SpanKind = 'internal' | 'client'; @@ -52,6 +52,48 @@ export function recordSpanError(span: Span, error: unknown): void { span.status = { code: 'error', message: error instanceof Error ? error.message : String(error) }; } +/** Cost and estimation of the finished child spans of one open span (LOU-D48). */ +interface UsageRollup { + costUsd: number; + /** At least one finished child carried a cost. */ + priced: boolean; + /** A finished child had token usage but no known price. */ + unpriced: boolean; + estimated: boolean; +} + +/** Open spans by id; a span's finished children add their usage here. */ +const rollups = new Map(); + +/** + * Adds a finished span's usage to its parent's rollup: its `loushy.cost_usd` + * (or, with tokens but no price, marks the parent as unpriced), and whether + * any of it was estimated. + */ +function rollUpUsage(span: Span, own: UsageRollup): void { + const parent = span.parentId ? rollups.get(span.parentId) : undefined; + if (!parent) return; + const cost = span.attributes[SdkAttr.COST_USD]; + if (typeof cost === 'number') { + parent.costUsd += cost; + parent.priced = true; + } else if (span.attributes[GenAiAttr.USAGE_INPUT_TOKENS] !== undefined || own.unpriced) { + parent.unpriced = true; + } + if (span.attributes[SdkAttr.USAGE_ESTIMATED] === true) parent.estimated = true; +} + +/** Stamps the cumulative cost (and estimated flag) of a span's children onto it. */ +function applyRollup(span: Span, own: UsageRollup): void { + const priced = own.priced && !own.unpriced; + if (!priced && !own.estimated) return; + span.attributes = { + ...span.attributes, + ...(priced ? { [SdkAttr.COST_USD]: own.costUsd } : {}), + ...(own.estimated ? { [SdkAttr.USAGE_ESTIMATED]: true } : {}), + }; +} + /** * Receives span lifecycle notifications. No default implementation ships * in this SDK - consumers bridge onSpanStart/onSpanEnd into whatever @@ -80,6 +122,10 @@ export interface TraceExporter { * - `fn` receives the span so callers can read its generated `id` (e.g. * to pass as the `parentId` of a further-nested withSpan() call). * - `kind` is the OpenTelemetry span kind (default `internal`). + * - Before `onSpanEnd`, a span whose child spans carried `loushy.cost_usd` + * gets the sum as its own (cumulative) `loushy.cost_usd`, absent when a + * child with token usage had no known price, and `loushy.usage.estimated` + * when any child's tokens were estimated (LOU-D48). */ export async function withSpan( exporter: TraceExporter | undefined, @@ -98,6 +144,8 @@ export async function withSpan( ...(kind ? { kind } : {}), }; + const own: UsageRollup = { costUsd: 0, priced: false, unpriced: false, estimated: false }; + rollups.set(span.id, own); exporter?.onSpanStart(span); try { @@ -107,6 +155,9 @@ export async function withSpan( throw error; } finally { span.endTime = Date.now(); + rollups.delete(span.id); + applyRollup(span, own); + rollUpUsage(span, own); exporter?.onSpanEnd(span); } }