fix(tracing): scrub error text like content and widen scrubber patterns

CHANGELOG.md

@@ -67,6 +67,7 @@ opt-out.
 - The observe daemon refuses cross-site browser writes and DNS rebinding on `/api/*`: a loopback-bound daemon answers only loopback `Host` names (403), writes carrying a non-loopback `Origin` (including `null`) or `Sec-Fetch-Site: cross-site` get 403, and a write with a body must be `Content-Type: application/json` (415) — `curl -d` needs `-H "Content-Type: application/json"`. The SDK reporter, CLI and Studio already comply. See [Observability](docs/observability.md).
 - Observe retention (`lousho observe start --retention-days`, `lousho observe prune`) also deletes old traces the sweep marked `interrupted` (a crashed run whose end never arrived); their age counts from when they were interrupted.
 - The observe daemon's online evaluator worker no longer blocks the daemon: it yields between traces, checks each rule's filter against the root row before loading anything else, hands code rules at most 200 observations (attrs over 8192 characters become `{ truncated: true }`) and no scores, and judge jobs claimed by Agent Forge carry the root fields only (`observations: []`).
+- Error text recorded on spans (`error` attributes and the span status message, on runs, LLM calls, tools, resumed runs and flows) goes through the same `redactContent` choice as captured content, so tokens in an error no longer reach traces, exports, evaluators or judge prompts; the built-in scrubber also catches case-insensitive `Bearer`, Stripe `sk_`/`rk_` live and test keys, Slack `xox[abprs]-` tokens, Google `AIza` keys and JWTs. `recordSpanError` and `withSpan` take an optional redactor (default: the built-in scrubber).
 
 ### Documentation
 

api/index.api.md

@@ -5144,7 +5144,7 @@ interface ReceiptEntry {
 }
 
 // @public
-export function recordSpanError(span: Span, error: unknown): void;
+export function recordSpanError(span: Span, error: unknown, redact?: (text: string) => string): void;
 
 // @public
 export interface RecoverOptions<TResult = unknown> {
@@ -7398,7 +7398,7 @@ export interface WithRetryOptions {
 }
 
 // @public
-export function withSpan<T>(exporter: TraceExporter | undefined, name: string, attributes: Record<string, unknown>, fn: (span: Span) => Promise<T>, parentId?: string, kind?: SpanKind): Promise<T>;
+export function withSpan<T>(exporter: TraceExporter | undefined, name: string, attributes: Record<string, unknown>, fn: (span: Span) => Promise<T>, parentId?: string, kind?: SpanKind, redact?: (text: string) => string): Promise<T>;
 
 // @public
 export function withSubagentOptions<T extends Subagents>(subagents: T, options: SubagentOptions): T;

docs/observability.md

@@ -263,10 +263,18 @@ with `[redacted]`:
 | `lsv2_…` (LangSmith) | `lsv2_pt_0123…` |
 | `ghp_…`, `gho_…`, `ghu_…`, `ghs_…`, `ghr_…`, `github_pat_…` (GitHub) | `ghp_0123…` |
 | `AKIA` + 16 uppercase letters or digits (AWS access key id) | `AKIA…EXAMPLE` |
-| `Bearer <token>` (the word stays: `Bearer [redacted]`) | `Bearer eyJhbGci…` |
-
-Each shape needs a long token (16+ characters) after its prefix, so words like
-`sk-learn` or `task-` are left alone. The scrubber catches common key formats,
+| `Bearer <token>` in any case (the word stays as written: `Bearer [redacted]`) | `Bearer <token>` |
+| `sk_live_…`, `rk_live_…`, `sk_test_…`, `rk_test_…` (Stripe) | `<stripe-key>` |
+| `xoxb-…`, `xoxa-…`, `xoxp-…`, `xoxr-…`, `xoxs-…` (Slack) | `<slack-token>` |
+| `AIza` + 35 characters (Google API key) | `<google-api-key>` |
+| `eyJ….eyJ….<signature>` (a JWT) | `<jwt>` |
+
+The same scrubbing (or your function) runs on the error text recorded on a span, `error` and the error status message, so a token in a thrown error's message is redacted too; `error.type` (the class name) is kept. `redactContent: false` records error text as is.
+
+Each shape needs a long token after its prefix (16+ characters for `sk-`, `lsv2_`,
+`gh*_`, `github_pat_`, Stripe and Bearer; 10+ for Slack; 35 for Google; JWT parts of 8+), so
+words like `sk-learn` or `task-` are left alone. Bearer is case-insensitive, so a long
+word after "bearer" in prose can be redacted too. The scrubber catches common key formats,
 not every secret: a password typed into a prompt is still recorded, so set
 `captureContent: false` if content must never reach the local db.
 

llms-full.txt

@@ -14491,10 +14491,18 @@ with `[redacted]`:
 | `lsv2_…` (LangSmith) | `lsv2_pt_0123…` |
 | `ghp_…`, `gho_…`, `ghu_…`, `ghs_…`, `ghr_…`, `github_pat_…` (GitHub) | `ghp_0123…` |
 | `AKIA` + 16 uppercase letters or digits (AWS access key id) | `AKIA…EXAMPLE` |
-| `Bearer <token>` (the word stays: `Bearer [redacted]`) | `Bearer eyJhbGci…` |
-
-Each shape needs a long token (16+ characters) after its prefix, so words like
-`sk-learn` or `task-` are left alone. The scrubber catches common key formats,
+| `Bearer <token>` in any case (the word stays as written: `Bearer [redacted]`) | `Bearer <token>` |
+| `sk_live_…`, `rk_live_…`, `sk_test_…`, `rk_test_…` (Stripe) | `<stripe-key>` |
+| `xoxb-…`, `xoxa-…`, `xoxp-…`, `xoxr-…`, `xoxs-…` (Slack) | `<slack-token>` |
+| `AIza` + 35 characters (Google API key) | `<google-api-key>` |
+| `eyJ….eyJ….<signature>` (a JWT) | `<jwt>` |
+
+The same scrubbing (or your function) runs on the error text recorded on a span, `error` and the error status message, so a token in a thrown error's message is redacted too; `error.type` (the class name) is kept. `redactContent: false` records error text as is.
+
+Each shape needs a long token after its prefix (16+ characters for `sk-`, `lsv2_`,
+`gh*_`, `github_pat_`, Stripe and Bearer; 10+ for Slack; 35 for Google; JWT parts of 8+), so
+words like `sk-learn` or `task-` are left alone. Bearer is case-insensitive, so a long
+word after "bearer" in prose can be redacted too. The scrubber catches common key formats,
 not every secret: a password typed into a prompt is still recorded, so set
 `captureContent: false` if content must never reach the local db.
 

src/execution/AgentExecutor.ts

@@ -29,6 +29,7 @@ import { TraceExporter, withSpan } from './tracing';
 import { withObserveExporter } from '../observe/reporter';
 import {
   agentRunSpanInit,
+  contentRedactor,
   recordToolOutcome,
   resolveCaptureContent,
   toolSpanInit,
@@ -870,7 +871,8 @@ export class AgentExecutor {
         init.attributes,
         async (agentSpan) => this.runWithEnd(observed, agentSpan.id),
         observed.parentSpanId,
-        init.kind
+        init.kind,
+        contentRedactor(observed)
       );
     });
   }
@@ -1840,7 +1842,8 @@ export class AgentExecutor {
         return executed;
       },
       agentSpanId,
-      init.kind
+      init.kind,
+      contentRedactor({ redactContent })
     );
   }
 

src/execution/codeMode.ts

@@ -31,7 +31,7 @@ import { runToolCall, type ToolCallContext, type ToolCallOutcome } from './toolC
 import { Limiter, type ToolConcurrency } from './toolBatch';
 import { runEventsOf } from './agentRun';
 import { withSpan, type TraceExporter } from './tracing';
-import { recordToolOutcome, resolveCaptureContent, toolSpanInit } from './genAiSpans';
+import { contentRedactor, recordToolOutcome, resolveCaptureContent, toolSpanInit } from './genAiSpans';
 import { SdkAttr } from './semconv';
 import { jsonSchemaOf } from './toolSearch';
 import { TOOL_SEARCH_TOOL, type ToolDeferral } from './toolDeferral';
@@ -347,7 +347,8 @@ export function nestedToolCaller(setup: NestedCallSetup): NestedToolCaller {
           return settled;
         },
         setup.parentSpanId,
-        init.kind
+        init.kind,
+        contentRedactor({ redactContent: tracing.redactContent })
       );
       sink?.toolSettled(outcome, parent);
       return outcome.error === undefined ? { json: toJson(outcome.result) } : { error: outcome.error };

src/execution/genAiSemconv.test.ts

@@ -345,6 +345,84 @@ describe('OpenTelemetry GenAI semantic conventions', () => {
     expect(JSON.parse(spans[0].attributes['gen_ai.input.messages'] as string)[0].parts[0].content).toBe('[redacted]');
   });
 
+  describe('tool error text', () => {
+    const secret = 'abcdefghij0123456789ABCD';
+    const token = 'Bearer ' + secret;
+    const failing = defineTool({
+      name: 'get_weather',
+      description: 'Get the weather for a city',
+      input: z.object({ city: z.string() }),
+      execute: async () => {
+        throw new Error(`upstream rejected ${token}`);
+      },
+    });
+
+    async function failedToolSpan(redactContent?: boolean | ((text: string) => string)) {
+      const { agent } = setup();
+      const toolRegistry = new ToolRegistry();
+      toolRegistry.register(failing);
+      const { exporter, spans } = memoryExporter();
+      await AgentExecutor.execute({
+        agent,
+        input: 'Weather?',
+        provider: modelWithOneToolCall(),
+        toolRegistry,
+        exporter,
+        ...(redactContent === undefined ? {} : { redactContent }),
+      });
+      return spans.find((s) => s.name.startsWith('execute_tool'))!;
+    }
+
+    it('scrubs the error message on a failed tool span by default', async () => {
+      const tool = await failedToolSpan();
+      expect(tool.attributes.error).toContain('Bearer [redacted]');
+      expect(JSON.stringify([tool.attributes, tool.status])).not.toContain(secret);
+      expect(tool.status?.message).toContain('Bearer [redacted]');
+    });
+
+    it('keeps the error message under redactContent: false', async () => {
+      const tool = await failedToolSpan(false);
+      expect(tool.attributes.error).toContain(token);
+      expect(tool.status?.message).toContain(token);
+    });
+
+    it('applies a custom redactor to the error message', async () => {
+      const tool = await failedToolSpan((text) => text.replace('upstream', 'remote'));
+      expect(tool.status?.message).toContain('remote rejected');
+    });
+
+    it.each([
+      [false, true],
+      [undefined, false],
+    ])('redactContent %s keeps (%s) the provider error text on the run span', async (redactContent, keeps) => {
+      const { agent, toolRegistry } = setup();
+      const { exporter, spans } = memoryExporter();
+      const provider = mockModel([]);
+      provider.generate = async () => {
+        throw new Error(`provider said ${token}`);
+      };
+      await AgentExecutor.execute({ agent, input: 'hi', provider, toolRegistry, exporter, ...(redactContent === undefined ? {} : { redactContent }) }).catch(() => undefined);
+      const run = spans.find((s) => s.name.startsWith('invoke_agent'))!;
+      const chat = spans.find((s) => s.name.startsWith('chat'))!;
+      for (const span of [run, chat]) {
+        expect(JSON.stringify([span.attributes.error, span.status]).includes(secret)).toBe(keeps);
+      }
+    });
+
+    it('scrubs the error recorded on the run span when the run itself fails', async () => {
+      const { agent, toolRegistry } = setup();
+      const { exporter, spans } = memoryExporter();
+      const provider = mockModel([]);
+      provider.generate = async () => {
+        throw new Error(`provider said ${token}`);
+      };
+      await AgentExecutor.execute({ agent, input: 'hi', provider, toolRegistry, exporter }).catch(() => undefined);
+      const text = JSON.stringify(spans.map((s) => [s.attributes.error, s.status]));
+      expect(text).toContain('provider said');
+      expect(text).not.toContain(secret);
+    });
+  });
+
   it('does not scrub captured content when redactContent is false', async () => {
     const key = 'sk-or-v1-0123456789abcdef0123456789abcdef';
     const { agent, toolRegistry } = setup();

src/execution/genAiSpans.ts

@@ -26,7 +26,7 @@ import type { Span, SpanKind } from './tracing';
 import { estimateCost } from '../models';
 import { normalizeUsage } from '../models/usage';
 import type { Usage } from '../models/usage';
-import { REDACTED, scrubSecrets } from './scrubSecrets';
+import { safeRedactor, scrubSecrets } from './scrubSecrets';
 
 /** Name, kind and starting attributes for a span about to be opened. */
 interface SpanInit {
@@ -64,19 +64,8 @@ type Redact = (text: string) => string;
 
 const keepText: Redact = (text) => text;
 
-/** A custom redactor that throws redacts that text whole: telemetry never fails a run. */
-function safeRedactor(redact: Redact): Redact {
-  return (text) => {
-    try {
-      return redact(text);
-    } catch {
-      return REDACTED;
-    }
-  };
-}
-
 /** The text mapping `redactContent` asks for on captured `gen_ai.*` content. */
-function contentRedactor(content: ContentOptions): Redact {
+export function contentRedactor(content: ContentOptions): Redact {
   const { redactContent } = content;
   if (redactContent === false) return keepText;
   return typeof redactContent === 'function' ? safeRedactor(redactContent) : scrubSecrets;
@@ -345,7 +334,7 @@ export function recordToolOutcome(
     ...defined({
       ...(!keepLegacyContent(content) ? {} : { [LegacyAttr.ARGS]: outcome.args, [LegacyAttr.RESULT]: outcome.result }),
       // The message, as on every other span's `error`; absent when the tool succeeded.
-      [LegacyAttr.ERROR]: outcome.error,
+      [LegacyAttr.ERROR]: outcome.error === undefined ? undefined : redact(outcome.error),
       // The sandbox boundary's failure kind ('timeout' | 'oom' | 'driver'), absent for the tool's own errors.
       [SandboxAttr.ERROR]: outcome.sandboxError,
       [LegacyAttr.LATENCY_MS]: outcome.latencyMs,
@@ -356,6 +345,6 @@ export function recordToolOutcome(
     }),
   };
   if (outcome.error) {
-    span.status = { code: 'error', message: outcome.error };
+    span.status = { code: 'error', message: redact(outcome.error) };
   }
 }

src/execution/generateStep.ts

@@ -16,7 +16,7 @@ import { withSpan, type Span } from './tracing';
 import { withProviderEvents } from '../providers/providerEvents';
 import type { CallUsage } from '../models/usage';
 import { measureUsage } from './runUsage';
-import { llmSpanInit, recordLlmResult, resolveCaptureContent } from './genAiSpans';
+import { contentRedactor, llmSpanInit, recordLlmResult, resolveCaptureContent } from './genAiSpans';
 import { SdkAttr } from './semconv';
 import { GenerateHookContext } from './hooks';
 import { APICallError } from 'ai';
@@ -125,7 +125,8 @@ export function sideGenerator(
         return generated;
       },
       agentSpanId,
-      init.kind
+      init.kind,
+      contentRedactor({ redactContent })
     );
   };
 }
@@ -335,7 +336,8 @@ export function generateInSpan(
       return { generated, measured };
     },
     agentSpanId,
-    init.kind
+    init.kind,
+    contentRedactor({ redactContent })
   );
 }
 

src/execution/resume.test.ts

@@ -1290,6 +1290,35 @@ describe('Execution - resumeAfterApproval', () => {
   });
 });
 
+describe('Execution - resumeAfterApproval: error text on the resumed run span', () => {
+  const secret = 'abcdefghij0123456789ABCD';
+  const failedRunSpan = async (executeOptions: ResumeExecuteOptions) => {
+    const toolRegistry = new ToolRegistry();
+    toolRegistry.register('chargeCard', {
+      displayName: 'Charge Card',
+      tool: { description: 'Charge a card', parameters: {}, execute: async () => ({ charged: true }) } as Tool,
+      needsApproval: true,
+    });
+    const agent = AgentBuilder.create().setName('Test Agent').addTool('chargeCard', { tool: 'chargeCard', options: {} }).build();
+    const provider = createMockProvider({ name: 'mock', responses: ['Charging now', 'All done'] });
+    const approvalStore = createInMemoryApprovalStore();
+    const paused = await AgentExecutor.execute({ agent, input: 'Please call chargeCard now', provider, toolRegistry, approvalStore });
+    const failing = { ...provider, generate: async () => { throw new Error('boom Bearer ' + secret); } } as unknown as LLMProvider;
+    const ended: Span[] = [];
+    const exporter = { onSpanStart: () => {}, onSpanEnd: (span: Span) => void ended.push({ ...span }) };
+    await resumeAfterApproval({ id: paused.approvalId!, approved: true }, approvalStore, toolRegistry, failing, { exporter, ...executeOptions }).catch(() => undefined);
+    return ended.find((span) => span.name.startsWith('invoke_agent'))!;
+  };
+
+  it('scrubs by default, applies a custom redactor, keeps the text under redactContent: false', async () => {
+    expect(JSON.stringify((await failedRunSpan({})).status)).not.toContain(secret);
+    expect((await failedRunSpan({ redactContent: (t) => t.replace(secret, '<custom>') })).status?.message).toContain('<custom>');
+    const kept = await failedRunSpan({ redactContent: false });
+    expect(kept.status?.message).toContain(secret);
+    expect(kept.attributes.error).toContain(secret);
+  });
+});
+
 describe('Execution - resumeAfterApproval: the span of the approved call (#281)', () => {
   const approvedRun = async (executeOptions: ResumeExecuteOptions) => {
     const toolRegistry = new ToolRegistry();

src/execution/resume.ts

@@ -54,7 +54,7 @@ import { readonlyPrincipal } from './runPrincipal';
 import type { OAuthTokenStore } from '../oauth/types';
 import { isSignInRequired, settleSignInRequired, signInOwner, signInRequest, SignInPendingError, type SignInRequired } from '../oauth/signIn';
 import { newId } from '../utils/id';
-import { agentRunSpanInit, recordToolOutcome, resolveCaptureContent, toolSpanInit, SUBAGENT_SPAN, type SubagentSpanInfo } from './genAiSpans';
+import { agentRunSpanInit, contentRedactor, recordToolOutcome, resolveCaptureContent, toolSpanInit, SUBAGENT_SPAN, type SubagentSpanInfo } from './genAiSpans';
 import { sandboxErrorKind } from '../sandbox/errors';
 import { withSpan, type Span } from './tracing';
 
@@ -242,7 +242,7 @@ function continueDecided(resume: DecidedResume): Promise<ExecutionResult> {
     );
   const init = runSpan();
   const resumed: ResumedRun = { ctx, claimed, pending, drift, staleCheckpoint, checkpointStore, runSpan, lease };
-  return withSpan(executeOptions.exporter, init.name, init.attributes, (span) => continueInRunSpan(resumed, span), executeOptions.parentSpanId, init.kind);
+  return withSpan(executeOptions.exporter, init.name, init.attributes, (span) => continueInRunSpan(resumed, span), executeOptions.parentSpanId, init.kind, contentRedactor(executeOptions));
 }
 
 /**
@@ -1124,7 +1124,8 @@ async function runApprovedInSpan(ctx: ResumeContext, pending: PendingApproval, s
       }
     },
     ctx.runSpanId,
-    init.kind
+    init.kind,
+    contentRedactor({ redactContent })
   );
   if ('signIn' in outcome) throw outcome.signIn;
   return outcome.message;