From ac4c475dde5d6c8356f71e9f973352e7e793a34d Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Tue, 29 Sep 2026 02:04:30 +0300 Subject: [PATCH] LOU-K4: ship a bundled, opt-in OpenTelemetry TraceExporter Adds createOtelTraceExporter() (src/execution/otel.ts), exported via a new "@loushy/build-ai-agent/otel" subpath, so consumers no longer have to hand-roll the Span -> OTel span bridge shown in examples/tracing/run-otel.ts every time. It only depends on @opentelemetry/api (now an optional peer dependency, moved out of devDependencies), never a concrete OTel SDK/exporter backend, so the host app still supplies its own TracerProvider. - src/execution/otel.ts: createOtelTraceExporter({ tracerName, tracerVersion, tracer }) implementing TraceExporter, translating Span start/end + parentId into real OTel spans/context. - src/execution/otel.test.ts: unit tests against a fake OTel Tracer covering span naming/timing, attribute translation (primitives kept, objects stringified), span end/attribute merge, parent/child context threading, and the default trace.getTracer(...) resolution path. - package.json: new "./otel" exports entry; @opentelemetry/api moved to peerDependencies (optional) instead of devDependencies only. - tsup.config.ts: new execution/otel entry point, @opentelemetry/api externalized. - examples/tracing/run-otel.ts: now uses createOtelTraceExporter() instead of the local hand-rolled adapter; examples/tracing/otel-exporter.ts removed as its logic moved into src/execution/otel.ts. - docs/api-overview.md: documents the bundled exporter as the recommended path for real OTel integration, with TraceExporter/ withSpan() still documented as the underlying BYO-exporter interface. Co-Authored-By: Claude Sonnet 5 --- docs/api-overview.md | 6 ++ examples/tracing/console-exporter.ts | 3 +- examples/tracing/otel-exporter.ts | 81 -------------- examples/tracing/run-otel.ts | 17 ++- package-lock.json | 4 + package.json | 9 ++ src/execution/otel.test.ts | 153 +++++++++++++++++++++++++++ src/execution/otel.ts | 126 ++++++++++++++++++++++ tsup.config.ts | 3 +- 9 files changed, 315 insertions(+), 87 deletions(-) delete mode 100644 examples/tracing/otel-exporter.ts create mode 100644 src/execution/otel.test.ts create mode 100644 src/execution/otel.ts diff --git a/docs/api-overview.md b/docs/api-overview.md index 4082e3cd6..00f9eea45 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -49,6 +49,12 @@ npm run docs:build # writes docs/api/index.html - `defineEval()`, scorers such as `exactMatch` and `toolCallOrder`, and `llmJudge()` - agent evals run under vitest. - `withSpan()` and `TraceExporter` - tracing for `AgentExecutor.execute()`. + `TraceExporter` is a bring-your-own-exporter interface (no exporter + ships by default); for real OpenTelemetry spans, import + `createOtelTraceExporter()` from the `@loushy/build-ai-agent/otel` + subpath (requires the optional peer dependency `@opentelemetry/api`) + instead of hand-rolling the OTel bridge - see + `examples/tracing/run-otel.ts`. - `NoopSandbox` / `SubprocessSandbox` - sandboxing for tools that opt in via `requiresSandbox`; guardrails such as `createCommandGuardrail()` and `secretScanGuardrail`. diff --git a/examples/tracing/console-exporter.ts b/examples/tracing/console-exporter.ts index 528dccc50..1ef7954a2 100644 --- a/examples/tracing/console-exporter.ts +++ b/examples/tracing/console-exporter.ts @@ -5,7 +5,8 @@ * console when a span starts and another when it ends, indenting child * spans (matched via Span.parentId) under their parent for readability. * Useful for local development, or as a template for writing a bridge - * into a real tracing backend (see otel-exporter.ts for that). + * into a real tracing backend (see src/execution/otel.ts / + * createOtelTraceExporter() for a bundled OpenTelemetry bridge). */ import { Span, TraceExporter } from '../../src/execution/tracing'; diff --git a/examples/tracing/otel-exporter.ts b/examples/tracing/otel-exporter.ts deleted file mode 100644 index bdad402ad..000000000 --- a/examples/tracing/otel-exporter.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * OpenTelemetry TraceExporter bridge (LOU-E6) - * - * Bridges this SDK's Span/TraceExporter (src/execution/tracing.ts) into - * real OpenTelemetry spans, propagating parent/child relationships - * (Span.parentId) into OTel's context API so a real OTel backend (Jaeger, - * Tempo, Datadog, the console exporter used by run-otel.ts, ...) sees the - * same agent.run -> llm.generate/tool.call span tree. - * - * @opentelemetry/api and @opentelemetry/sdk-trace-node are DEV - * dependencies ONLY (see package.json) - this SDK does not require or - * ship OpenTelemetry as a runtime dependency. This file is an example of - * how a consumer who *does* want OTel can wire it up themselves. - */ - -import { trace, context, Context, Span as OtelSpan, Tracer } from '@opentelemetry/api'; -import { Span, TraceExporter } from '../../src/execution/tracing'; - -/** - * Creates a TraceExporter that starts/ends a real OTel span for every - * Span this SDK produces, using `tracer` (typically obtained from a - * configured NodeTracerProvider - see run-otel.ts). - */ -export function createOtelExporter(tracer: Tracer): TraceExporter { - // Our Span.id -> the OTel span + context it was started in, so a child - // Span (matched by parentId) can be started as a child of the right - // OTel context instead of the ambient one. - const otelSpansById = new Map(); - - return { - onSpanStart(span: Span) { - const parentEntry = span.parentId ? otelSpansById.get(span.parentId) : undefined; - const parentContext = parentEntry ? parentEntry.ctx : context.active(); - - const otelSpan = tracer.startSpan( - span.name, - { startTime: span.startTime }, - parentContext - ); - - // Seed OTel span attributes with whatever this SDK already knew at - // span-start time (e.g. model, toolName). onSpanEnd below adds - // whatever was filled in afterwards (token counts, results, ...). - setAttributes(otelSpan, span.attributes); - - const ctx = trace.setSpan(parentContext, otelSpan); - otelSpansById.set(span.id, { span: otelSpan, ctx }); - }, - - onSpanEnd(span: Span) { - const entry = otelSpansById.get(span.id); - if (!entry) { - return; - } - - setAttributes(entry.span, span.attributes); - entry.span.end(span.endTime); - otelSpansById.delete(span.id); - }, - }; -} - -function setAttributes(otelSpan: OtelSpan, attributes: Record): void { - for (const [key, value] of Object.entries(attributes)) { - if (value === undefined) { - continue; - } - // OTel attribute values must be a primitive or an array of a single - // primitive type - stringify anything else (objects/arrays of - // objects, e.g. `prompt`/`args`/`result`) so nothing throws. - if ( - typeof value === 'string' || - typeof value === 'number' || - typeof value === 'boolean' - ) { - otelSpan.setAttribute(key, value); - } else { - otelSpan.setAttribute(key, JSON.stringify(value)); - } - } -} diff --git a/examples/tracing/run-otel.ts b/examples/tracing/run-otel.ts index dd2d0ccfd..327c6d109 100644 --- a/examples/tracing/run-otel.ts +++ b/examples/tracing/run-otel.ts @@ -1,7 +1,13 @@ /** * Runnable example: AgentExecutor.execute() wired to a real OpenTelemetry * NodeTracerProvider, using OTel's own ConsoleSpanExporter so the emitted - * spans are visible without needing a real collector (LOU-E6). + * spans are visible without needing a real collector (LOU-E6, LOU-K4). + * + * The TraceExporter itself comes from the bundled, opt-in + * `createOtelTraceExporter()` (src/execution/otel.ts) - this example only + * wires up the OTel SDK plumbing (a NodeTracerProvider + span processor) + * that decides *where* the spans go; the SDK-to-OTel translation is no + * longer hand-rolled here. * * Run with: * npm run example:tracing:otel @@ -15,11 +21,11 @@ import { SimpleSpanProcessor, } from '@opentelemetry/sdk-trace-node'; import { AgentExecutor } from '../../src/execution/AgentExecutor'; +import { createOtelTraceExporter } from '../../src/execution/otel'; import { AgentBuilder } from '../../src/core'; import { AgentType } from '../../src/types'; import { ToolRegistry } from '../../src/tools'; import { LLMProvider, GenerateOptions, GenerateResult } from '../../src/providers'; -import { createOtelExporter } from './otel-exporter'; /** Same tiny mock provider as run-console.ts, so this needs no API key. */ class MockProvider implements LLMProvider { @@ -73,6 +79,11 @@ async function main() { provider.register(); const tracer = provider.getTracer('loushy-tracing-example'); + // In real usage the tracer name is enough - `createOtelTraceExporter()` + // will resolve one via `trace.getTracer(...)` for you. Passing the + // tracer explicitly here just reuses the one obtained above. + const exporter = createOtelTraceExporter({ tracer }); + const toolRegistry = new ToolRegistry(); toolRegistry.register('getWeather', { displayName: 'Get Weather', @@ -90,8 +101,6 @@ async function main() { .addTool('getWeather', { tool: 'getWeather', options: {} }) .build(); - const exporter = createOtelExporter(tracer); - const result = await AgentExecutor.execute({ agent, input: "What's the weather in Cairo?", diff --git a/package-lock.json b/package-lock.json index b4af80e98..a7bb76820 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "peerDependencies": { "@ai-sdk/anthropic": "^0.0.42", "@ai-sdk/openai": "^0.0.42", + "@opentelemetry/api": "^1.9.1", "ai": "^4.3.19", "ollama-ai-provider": "^1.2.0", "tsup": "^8.0.0", @@ -56,6 +57,9 @@ "@ai-sdk/openai": { "optional": true }, + "@opentelemetry/api": { + "optional": true + }, "ollama-ai-provider": { "optional": true }, diff --git a/package.json b/package.json index 83b23b615..fb974c448 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,11 @@ "types": "./dist/data/mocks.d.ts", "import": "./dist/data/mocks.mjs", "require": "./dist/data/mocks.js" + }, + "./otel": { + "types": "./dist/execution/otel.d.ts", + "import": "./dist/execution/otel.mjs", + "require": "./dist/execution/otel.js" } }, "files": [ @@ -105,6 +110,7 @@ "peerDependencies": { "@ai-sdk/anthropic": "^0.0.42", "@ai-sdk/openai": "^0.0.42", + "@opentelemetry/api": "^1.9.1", "ai": "^4.3.19", "ollama-ai-provider": "^1.2.0", "tsup": "^8.0.0", @@ -117,6 +123,9 @@ "@ai-sdk/openai": { "optional": true }, + "@opentelemetry/api": { + "optional": true + }, "ollama-ai-provider": { "optional": true }, diff --git a/src/execution/otel.test.ts b/src/execution/otel.test.ts new file mode 100644 index 000000000..2463686a9 --- /dev/null +++ b/src/execution/otel.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { Tracer, Span as OtelSpan, Context } from '@opentelemetry/api'; +import { createOtelTraceExporter } from './otel'; +import { Span } from './tracing'; + +/** + * A minimal fake OTel span that just records what was done to it, so + * assertions can check exact call shapes without needing a real + * TracerProvider/exporter backend. + */ +function makeFakeOtelSpan(name: string) { + return { + name, + attributes: {} as Record, + ended: false, + endTime: undefined as number | undefined, + setAttribute(key: string, value: unknown) { + this.attributes[key] = value; + return this; + }, + end(endTime?: number) { + this.ended = true; + this.endTime = endTime; + }, + } as unknown as OtelSpan & { + name: string; + attributes: Record; + ended: boolean; + endTime: number | undefined; + }; +} + +function makeFakeTracer() { + const startedSpans: Array<{ + name: string; + options: unknown; + parentContext: unknown; + span: ReturnType; + }> = []; + + const tracer: Tracer = { + startSpan: vi.fn((name: string, options: unknown, parentContext?: Context) => { + const span = makeFakeOtelSpan(name); + startedSpans.push({ name, options, parentContext, span }); + return span; + }), + startActiveSpan: vi.fn() as unknown as Tracer['startActiveSpan'], + }; + + return { tracer, startedSpans }; +} + +function makeSpan(overrides: Partial = {}): Span { + return { + id: 'span-1', + name: 'agent.run', + attributes: { model: 'gpt-4' }, + startTime: 1000, + ...overrides, + }; +} + +describe('createOtelTraceExporter', () => { + let fake: ReturnType; + + beforeEach(() => { + fake = makeFakeTracer(); + }); + + it('starts an OTel span with the SDK span name and startTime on onSpanStart', () => { + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + const span = makeSpan({ id: 's1', name: 'agent.run', startTime: 1234, attributes: { model: 'gpt-4' } }); + + exporter.onSpanStart(span); + + expect(fake.tracer.startSpan).toHaveBeenCalledTimes(1); + const [name, options] = (fake.tracer.startSpan as ReturnType).mock.calls[0]; + expect(name).toBe('agent.run'); + expect(options).toMatchObject({ startTime: 1234 }); + expect(fake.startedSpans[0].span.attributes).toEqual({ model: 'gpt-4' }); + }); + + it('translates primitive attributes with setAttribute and stringifies non-primitives', () => { + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + const span = makeSpan({ + id: 's1', + attributes: { + model: 'gpt-4', + tokens: 42, + cached: true, + args: { city: 'Cairo' }, + skip: undefined, + }, + }); + + exporter.onSpanStart(span); + + expect(fake.startedSpans[0].span.attributes).toEqual({ + model: 'gpt-4', + tokens: 42, + cached: true, + args: JSON.stringify({ city: 'Cairo' }), + }); + }); + + it('ends the matching OTel span with the SDK span endTime and merged attributes on onSpanEnd', () => { + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + const span = makeSpan({ id: 's1', attributes: { model: 'gpt-4' } }); + exporter.onSpanStart(span); + + const finished: Span = { ...span, endTime: 2000, attributes: { model: 'gpt-4', result: 'ok' } }; + exporter.onSpanEnd(finished); + + const otelSpan = fake.startedSpans[0].span; + expect(otelSpan.ended).toBe(true); + expect(otelSpan.endTime).toBe(2000); + expect(otelSpan.attributes).toEqual({ model: 'gpt-4', result: 'ok' }); + }); + + it('is a no-op on onSpanEnd for a span id that was never started', () => { + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + expect(() => + exporter.onSpanEnd(makeSpan({ id: 'unknown', endTime: 999 })) + ).not.toThrow(); + }); + + it('starts a child span using the parent OTel context when parentId matches a started span', () => { + const exporter = createOtelTraceExporter({ tracer: fake.tracer }); + const parent = makeSpan({ id: 'parent', name: 'agent.run' }); + exporter.onSpanStart(parent); + + const child = makeSpan({ id: 'child', name: 'llm.generate', parentId: 'parent', startTime: 1100 }); + exporter.onSpanStart(child); + + expect(fake.tracer.startSpan).toHaveBeenCalledTimes(2); + // The child's startSpan call should receive a parentContext distinct + // from the ambient/default context used for the root span - i.e. it + // was threaded from the parent's own context, not context.active(). + const rootParentContext = fake.startedSpans[0].parentContext; + const childParentContext = fake.startedSpans[1].parentContext; + expect(childParentContext).not.toBe(rootParentContext); + }); + + it('defaults to a tracer resolved via trace.getTracer when no tracer option is passed', async () => { + const otelApi = await import('@opentelemetry/api'); + const getTracerSpy = vi.spyOn(otelApi.trace, 'getTracer'); + + createOtelTraceExporter({ tracerName: 'my-agent', tracerVersion: '1.2.3' }); + + expect(getTracerSpy).toHaveBeenCalledWith('my-agent', '1.2.3'); + getTracerSpy.mockRestore(); + }); +}); diff --git a/src/execution/otel.ts b/src/execution/otel.ts new file mode 100644 index 000000000..b90c9da50 --- /dev/null +++ b/src/execution/otel.ts @@ -0,0 +1,126 @@ +/** + * Bundled, opt-in OpenTelemetry TraceExporter (LOU-K4). + * + * `tracing.ts` deliberately ships no bundled exporter - `withSpan()` / + * `TraceExporter` are a bring-your-own-exporter abstraction so this SDK + * never forces a tracing backend on every consumer. Historically, wiring + * that up to real OpenTelemetry meant hand-rolling the adapter shown in + * `examples/tracing/run-otel.ts` (LOU-E6) every time. + * + * This module is the ready-made version of that adapter: import + * `createOtelTraceExporter()` from the `@loushy/build-ai-agent/otel` + * subpath to translate this SDK's Span/TraceExporter shape into real + * OpenTelemetry spans via `@opentelemetry/api`'s `trace.getTracer(...)`, + * without pulling any OTel SDK/exporter backend into this package. + * + * `@opentelemetry/api` is an OPTIONAL peer dependency (see package.json) - + * it is only required at runtime by consumers who import this subpath. + * Importing `@loushy/build-ai-agent` (or any other subpath) never loads + * this module and never requires `@opentelemetry/api` to be installed. + * If it isn't installed, importing `@loushy/build-ai-agent/otel` throws a + * clear module-not-found error at that import site, not at package-import + * time. + */ + +import { trace, context, Context, Span as OtelSpan, Tracer } from '@opentelemetry/api'; +import { Span, TraceExporter } from './tracing'; + +/** + * Options for {@link createOtelTraceExporter}. + */ +export interface OtelTraceExporterOptions { + /** + * Name passed to `trace.getTracer(tracerName, tracerVersion)`. Defaults + * to `'@loushy/build-ai-agent'`. Ignored when `tracer` is supplied. + */ + tracerName?: string; + /** + * Version passed to `trace.getTracer(tracerName, tracerVersion)`. + * Ignored when `tracer` is supplied. + */ + tracerVersion?: string; + /** + * Use an already-obtained OTel `Tracer` instead of resolving one via + * `trace.getTracer(...)`. Useful when the host app manages its own + * `TracerProvider` registration and wants full control over which + * tracer instance is used. + */ + tracer?: Tracer; +} + +/** + * Creates a {@link TraceExporter} that starts/ends a real OpenTelemetry + * span for every `Span` this SDK produces, propagating parent/child + * relationships (`Span.parentId`) into OTel's context API so a real OTel + * backend sees the same `agent.run` -> `llm.generate`/`tool.call` span + * tree that `withSpan()` builds internally. + * + * This only depends on `@opentelemetry/api` (the OTel instrumentation + * API), not any concrete SDK/exporter backend - the host application is + * responsible for registering a `TracerProvider` (e.g. via + * `@opentelemetry/sdk-trace-node`) so the spans emitted here actually go + * somewhere. Without a registered provider, OTel's default no-op tracer + * is used and spans are silently discarded, which is safe but means + * nothing is exported. + * + * @example + * ```ts + * import { createOtelTraceExporter } from '@loushy/build-ai-agent/otel'; + * + * const exporter = createOtelTraceExporter({ tracerName: 'my-agent' }); + * await AgentExecutor.execute({ agent, input, provider, toolRegistry, exporter }); + * ``` + */ +export function createOtelTraceExporter(options: OtelTraceExporterOptions = {}): TraceExporter { + const tracer = + options.tracer ?? trace.getTracer(options.tracerName ?? '@loushy/build-ai-agent', options.tracerVersion); + + // Our Span.id -> the OTel span + context it was started in, so a child + // Span (matched by parentId) can be started as a child of the right + // OTel context instead of the ambient one. + const otelSpansById = new Map(); + + return { + onSpanStart(span: Span) { + const parentEntry = span.parentId ? otelSpansById.get(span.parentId) : undefined; + const parentContext = parentEntry ? parentEntry.ctx : context.active(); + + const otelSpan = tracer.startSpan(span.name, { startTime: span.startTime }, parentContext); + + // Seed OTel span attributes with whatever this SDK already knew at + // span-start time (e.g. model, toolName). onSpanEnd below adds + // whatever was filled in afterwards (token counts, results, ...). + setAttributes(otelSpan, span.attributes); + + const ctx = trace.setSpan(parentContext, otelSpan); + otelSpansById.set(span.id, { span: otelSpan, ctx }); + }, + + onSpanEnd(span: Span) { + const entry = otelSpansById.get(span.id); + if (!entry) { + return; + } + + setAttributes(entry.span, span.attributes); + entry.span.end(span.endTime); + otelSpansById.delete(span.id); + }, + }; +} + +function setAttributes(otelSpan: OtelSpan, attributes: Record): void { + for (const [key, value] of Object.entries(attributes)) { + if (value === undefined) { + continue; + } + // OTel attribute values must be a primitive or an array of a single + // primitive type - stringify anything else (objects/arrays of + // objects, e.g. `prompt`/`args`/`result`) so nothing throws. + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + otelSpan.setAttribute(key, value); + } else { + otelSpan.setAttribute(key, JSON.stringify(value)); + } + } +} diff --git a/tsup.config.ts b/tsup.config.ts index 97c706ad8..e717e7345 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -28,6 +28,7 @@ export default defineConfig({ 'data/mocks': 'src/data/mocks.ts', 'cli/dev': 'src/cli/dev.ts', 'cli/build': 'src/cli/build.ts', + 'execution/otel': 'src/execution/otel.ts', }, format: ['cjs', 'esm'], dts: true, @@ -36,7 +37,7 @@ export default defineConfig({ clean: true, treeshake: true, minify: false, - external: ['ai', 'zod'], + external: ['ai', 'zod', '@opentelemetry/api'], onSuccess: async () => { copyDevUi(); },