diff --git a/CHANGELOG.md b/CHANGELOG.md index 581b3ad19..9118b559e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ opt-out. - The observe daemon refuses cross-site browser writes and DNS rebinding on `/api/*`: a loopback-bound daemon answers only loopback `Host` names (403), writes carrying a non-loopback `Origin` (including `null`) or `Sec-Fetch-Site: cross-site` get 403, and a write with a body must be `Content-Type: application/json` (415) — `curl -d` needs `-H "Content-Type: application/json"`. The SDK reporter, CLI and Studio already comply. See [Observability](docs/observability.md). - Observe retention (`lousho observe start --retention-days`, `lousho observe prune`) also deletes old traces the sweep marked `interrupted` (a crashed run whose end never arrived); their age counts from when they were interrupted. - The observe daemon's online evaluator worker no longer blocks the daemon: it yields between traces, checks each rule's filter against the root row before loading anything else, hands code rules at most 200 observations (attrs over 8192 characters become `{ truncated: true }`) and no scores, and judge jobs claimed by Agent Forge carry the root fields only (`observations: []`). +- Error text recorded on spans (`error` attributes and the span status message, on runs, LLM calls, tools, resumed runs and flows) goes through the same `redactContent` choice as captured content, so tokens in an error no longer reach traces, exports, evaluators or judge prompts; the built-in scrubber also catches case-insensitive `Bearer`, Stripe `sk_`/`rk_` live and test keys, Slack `xox[abprs]-` tokens, Google `AIza` keys and JWTs. `recordSpanError` and `withSpan` take an optional redactor (default: the built-in scrubber). ### Documentation diff --git a/api/index.api.md b/api/index.api.md index 528fbd71a..0eb7d1c38 100644 --- a/api/index.api.md +++ b/api/index.api.md @@ -5144,7 +5144,7 @@ interface ReceiptEntry { } // @public -export function recordSpanError(span: Span, error: unknown): void; +export function recordSpanError(span: Span, error: unknown, redact?: (text: string) => string): void; // @public export interface RecoverOptions { @@ -7398,7 +7398,7 @@ export interface WithRetryOptions { } // @public -export function withSpan(exporter: TraceExporter | undefined, name: string, attributes: Record, fn: (span: Span) => Promise, parentId?: string, kind?: SpanKind): Promise; +export function withSpan(exporter: TraceExporter | undefined, name: string, attributes: Record, fn: (span: Span) => Promise, parentId?: string, kind?: SpanKind, redact?: (text: string) => string): Promise; // @public export function withSubagentOptions(subagents: T, options: SubagentOptions): T; diff --git a/docs/observability.md b/docs/observability.md index 9cb035387..d7f5abb9b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -263,10 +263,18 @@ with `[redacted]`: | `lsv2_…` (LangSmith) | `lsv2_pt_0123…` | | `ghp_…`, `gho_…`, `ghu_…`, `ghs_…`, `ghr_…`, `github_pat_…` (GitHub) | `ghp_0123…` | | `AKIA` + 16 uppercase letters or digits (AWS access key id) | `AKIA…EXAMPLE` | -| `Bearer ` (the word stays: `Bearer [redacted]`) | `Bearer eyJhbGci…` | - -Each shape needs a long token (16+ characters) after its prefix, so words like -`sk-learn` or `task-` are left alone. The scrubber catches common key formats, +| `Bearer ` in any case (the word stays as written: `Bearer [redacted]`) | `Bearer ` | +| `sk_live_…`, `rk_live_…`, `sk_test_…`, `rk_test_…` (Stripe) | `` | +| `xoxb-…`, `xoxa-…`, `xoxp-…`, `xoxr-…`, `xoxs-…` (Slack) | `` | +| `AIza` + 35 characters (Google API key) | `` | +| `eyJ….eyJ….` (a JWT) | `` | + +The same scrubbing (or your function) runs on the error text recorded on a span, `error` and the error status message, so a token in a thrown error's message is redacted too; `error.type` (the class name) is kept. `redactContent: false` records error text as is. + +Each shape needs a long token after its prefix (16+ characters for `sk-`, `lsv2_`, +`gh*_`, `github_pat_`, Stripe and Bearer; 10+ for Slack; 35 for Google; JWT parts of 8+), so +words like `sk-learn` or `task-` are left alone. Bearer is case-insensitive, so a long +word after "bearer" in prose can be redacted too. The scrubber catches common key formats, not every secret: a password typed into a prompt is still recorded, so set `captureContent: false` if content must never reach the local db. diff --git a/llms-full.txt b/llms-full.txt index dd59862dc..240636469 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -14491,10 +14491,18 @@ with `[redacted]`: | `lsv2_…` (LangSmith) | `lsv2_pt_0123…` | | `ghp_…`, `gho_…`, `ghu_…`, `ghs_…`, `ghr_…`, `github_pat_…` (GitHub) | `ghp_0123…` | | `AKIA` + 16 uppercase letters or digits (AWS access key id) | `AKIA…EXAMPLE` | -| `Bearer ` (the word stays: `Bearer [redacted]`) | `Bearer eyJhbGci…` | - -Each shape needs a long token (16+ characters) after its prefix, so words like -`sk-learn` or `task-` are left alone. The scrubber catches common key formats, +| `Bearer ` in any case (the word stays as written: `Bearer [redacted]`) | `Bearer ` | +| `sk_live_…`, `rk_live_…`, `sk_test_…`, `rk_test_…` (Stripe) | `` | +| `xoxb-…`, `xoxa-…`, `xoxp-…`, `xoxr-…`, `xoxs-…` (Slack) | `` | +| `AIza` + 35 characters (Google API key) | `` | +| `eyJ….eyJ….` (a JWT) | `` | + +The same scrubbing (or your function) runs on the error text recorded on a span, `error` and the error status message, so a token in a thrown error's message is redacted too; `error.type` (the class name) is kept. `redactContent: false` records error text as is. + +Each shape needs a long token after its prefix (16+ characters for `sk-`, `lsv2_`, +`gh*_`, `github_pat_`, Stripe and Bearer; 10+ for Slack; 35 for Google; JWT parts of 8+), so +words like `sk-learn` or `task-` are left alone. Bearer is case-insensitive, so a long +word after "bearer" in prose can be redacted too. The scrubber catches common key formats, not every secret: a password typed into a prompt is still recorded, so set `captureContent: false` if content must never reach the local db. diff --git a/src/execution/AgentExecutor.ts b/src/execution/AgentExecutor.ts index c1113c9c7..74a34465a 100644 --- a/src/execution/AgentExecutor.ts +++ b/src/execution/AgentExecutor.ts @@ -29,6 +29,7 @@ import { TraceExporter, withSpan } from './tracing'; import { withObserveExporter } from '../observe/reporter'; import { agentRunSpanInit, + contentRedactor, recordToolOutcome, resolveCaptureContent, toolSpanInit, @@ -870,7 +871,8 @@ export class AgentExecutor { init.attributes, async (agentSpan) => this.runWithEnd(observed, agentSpan.id), observed.parentSpanId, - init.kind + init.kind, + contentRedactor(observed) ); }); } @@ -1840,7 +1842,8 @@ export class AgentExecutor { return executed; }, agentSpanId, - init.kind + init.kind, + contentRedactor({ redactContent }) ); } diff --git a/src/execution/codeMode.ts b/src/execution/codeMode.ts index f13edd9f9..918a3bae7 100644 --- a/src/execution/codeMode.ts +++ b/src/execution/codeMode.ts @@ -31,7 +31,7 @@ import { runToolCall, type ToolCallContext, type ToolCallOutcome } from './toolC import { Limiter, type ToolConcurrency } from './toolBatch'; import { runEventsOf } from './agentRun'; import { withSpan, type TraceExporter } from './tracing'; -import { recordToolOutcome, resolveCaptureContent, toolSpanInit } from './genAiSpans'; +import { contentRedactor, recordToolOutcome, resolveCaptureContent, toolSpanInit } from './genAiSpans'; import { SdkAttr } from './semconv'; import { jsonSchemaOf } from './toolSearch'; import { TOOL_SEARCH_TOOL, type ToolDeferral } from './toolDeferral'; @@ -347,7 +347,8 @@ export function nestedToolCaller(setup: NestedCallSetup): NestedToolCaller { return settled; }, setup.parentSpanId, - init.kind + init.kind, + contentRedactor({ redactContent: tracing.redactContent }) ); sink?.toolSettled(outcome, parent); return outcome.error === undefined ? { json: toJson(outcome.result) } : { error: outcome.error }; diff --git a/src/execution/genAiSemconv.test.ts b/src/execution/genAiSemconv.test.ts index 683b726d0..ce77d0653 100644 --- a/src/execution/genAiSemconv.test.ts +++ b/src/execution/genAiSemconv.test.ts @@ -345,6 +345,84 @@ describe('OpenTelemetry GenAI semantic conventions', () => { expect(JSON.parse(spans[0].attributes['gen_ai.input.messages'] as string)[0].parts[0].content).toBe('[redacted]'); }); + describe('tool error text', () => { + const secret = 'abcdefghij0123456789ABCD'; + const token = 'Bearer ' + secret; + const failing = defineTool({ + name: 'get_weather', + description: 'Get the weather for a city', + input: z.object({ city: z.string() }), + execute: async () => { + throw new Error(`upstream rejected ${token}`); + }, + }); + + async function failedToolSpan(redactContent?: boolean | ((text: string) => string)) { + const { agent } = setup(); + const toolRegistry = new ToolRegistry(); + toolRegistry.register(failing); + const { exporter, spans } = memoryExporter(); + await AgentExecutor.execute({ + agent, + input: 'Weather?', + provider: modelWithOneToolCall(), + toolRegistry, + exporter, + ...(redactContent === undefined ? {} : { redactContent }), + }); + return spans.find((s) => s.name.startsWith('execute_tool'))!; + } + + it('scrubs the error message on a failed tool span by default', async () => { + const tool = await failedToolSpan(); + expect(tool.attributes.error).toContain('Bearer [redacted]'); + expect(JSON.stringify([tool.attributes, tool.status])).not.toContain(secret); + expect(tool.status?.message).toContain('Bearer [redacted]'); + }); + + it('keeps the error message under redactContent: false', async () => { + const tool = await failedToolSpan(false); + expect(tool.attributes.error).toContain(token); + expect(tool.status?.message).toContain(token); + }); + + it('applies a custom redactor to the error message', async () => { + const tool = await failedToolSpan((text) => text.replace('upstream', 'remote')); + expect(tool.status?.message).toContain('remote rejected'); + }); + + it.each([ + [false, true], + [undefined, false], + ])('redactContent %s keeps (%s) the provider error text on the run span', async (redactContent, keeps) => { + const { agent, toolRegistry } = setup(); + const { exporter, spans } = memoryExporter(); + const provider = mockModel([]); + provider.generate = async () => { + throw new Error(`provider said ${token}`); + }; + await AgentExecutor.execute({ agent, input: 'hi', provider, toolRegistry, exporter, ...(redactContent === undefined ? {} : { redactContent }) }).catch(() => undefined); + const run = spans.find((s) => s.name.startsWith('invoke_agent'))!; + const chat = spans.find((s) => s.name.startsWith('chat'))!; + for (const span of [run, chat]) { + expect(JSON.stringify([span.attributes.error, span.status]).includes(secret)).toBe(keeps); + } + }); + + it('scrubs the error recorded on the run span when the run itself fails', async () => { + const { agent, toolRegistry } = setup(); + const { exporter, spans } = memoryExporter(); + const provider = mockModel([]); + provider.generate = async () => { + throw new Error(`provider said ${token}`); + }; + await AgentExecutor.execute({ agent, input: 'hi', provider, toolRegistry, exporter }).catch(() => undefined); + const text = JSON.stringify(spans.map((s) => [s.attributes.error, s.status])); + expect(text).toContain('provider said'); + expect(text).not.toContain(secret); + }); + }); + it('does not scrub captured content when redactContent is false', async () => { const key = 'sk-or-v1-0123456789abcdef0123456789abcdef'; const { agent, toolRegistry } = setup(); diff --git a/src/execution/genAiSpans.ts b/src/execution/genAiSpans.ts index 3bfb0765f..acc47dc33 100644 --- a/src/execution/genAiSpans.ts +++ b/src/execution/genAiSpans.ts @@ -26,7 +26,7 @@ import type { Span, SpanKind } from './tracing'; import { estimateCost } from '../models'; import { normalizeUsage } from '../models/usage'; import type { Usage } from '../models/usage'; -import { REDACTED, scrubSecrets } from './scrubSecrets'; +import { safeRedactor, scrubSecrets } from './scrubSecrets'; /** Name, kind and starting attributes for a span about to be opened. */ interface SpanInit { @@ -64,19 +64,8 @@ type Redact = (text: string) => string; const keepText: Redact = (text) => text; -/** A custom redactor that throws redacts that text whole: telemetry never fails a run. */ -function safeRedactor(redact: Redact): Redact { - return (text) => { - try { - return redact(text); - } catch { - return REDACTED; - } - }; -} - /** The text mapping `redactContent` asks for on captured `gen_ai.*` content. */ -function contentRedactor(content: ContentOptions): Redact { +export function contentRedactor(content: ContentOptions): Redact { const { redactContent } = content; if (redactContent === false) return keepText; return typeof redactContent === 'function' ? safeRedactor(redactContent) : scrubSecrets; @@ -345,7 +334,7 @@ export function recordToolOutcome( ...defined({ ...(!keepLegacyContent(content) ? {} : { [LegacyAttr.ARGS]: outcome.args, [LegacyAttr.RESULT]: outcome.result }), // The message, as on every other span's `error`; absent when the tool succeeded. - [LegacyAttr.ERROR]: outcome.error, + [LegacyAttr.ERROR]: outcome.error === undefined ? undefined : redact(outcome.error), // The sandbox boundary's failure kind ('timeout' | 'oom' | 'driver'), absent for the tool's own errors. [SandboxAttr.ERROR]: outcome.sandboxError, [LegacyAttr.LATENCY_MS]: outcome.latencyMs, @@ -356,6 +345,6 @@ export function recordToolOutcome( }), }; if (outcome.error) { - span.status = { code: 'error', message: outcome.error }; + span.status = { code: 'error', message: redact(outcome.error) }; } } diff --git a/src/execution/generateStep.ts b/src/execution/generateStep.ts index 09fb286fd..6dd4cb834 100644 --- a/src/execution/generateStep.ts +++ b/src/execution/generateStep.ts @@ -16,7 +16,7 @@ import { withSpan, type Span } from './tracing'; import { withProviderEvents } from '../providers/providerEvents'; import type { CallUsage } from '../models/usage'; import { measureUsage } from './runUsage'; -import { llmSpanInit, recordLlmResult, resolveCaptureContent } from './genAiSpans'; +import { contentRedactor, llmSpanInit, recordLlmResult, resolveCaptureContent } from './genAiSpans'; import { SdkAttr } from './semconv'; import { GenerateHookContext } from './hooks'; import { APICallError } from 'ai'; @@ -125,7 +125,8 @@ export function sideGenerator( return generated; }, agentSpanId, - init.kind + init.kind, + contentRedactor({ redactContent }) ); }; } @@ -335,7 +336,8 @@ export function generateInSpan( return { generated, measured }; }, agentSpanId, - init.kind + init.kind, + contentRedactor({ redactContent }) ); } diff --git a/src/execution/resume.test.ts b/src/execution/resume.test.ts index 73fce1773..ec24232ad 100644 --- a/src/execution/resume.test.ts +++ b/src/execution/resume.test.ts @@ -1290,6 +1290,35 @@ describe('Execution - resumeAfterApproval', () => { }); }); +describe('Execution - resumeAfterApproval: error text on the resumed run span', () => { + const secret = 'abcdefghij0123456789ABCD'; + const failedRunSpan = async (executeOptions: ResumeExecuteOptions) => { + const toolRegistry = new ToolRegistry(); + toolRegistry.register('chargeCard', { + displayName: 'Charge Card', + tool: { description: 'Charge a card', parameters: {}, execute: async () => ({ charged: true }) } as Tool, + needsApproval: true, + }); + const agent = AgentBuilder.create().setName('Test Agent').addTool('chargeCard', { tool: 'chargeCard', options: {} }).build(); + const provider = createMockProvider({ name: 'mock', responses: ['Charging now', 'All done'] }); + const approvalStore = createInMemoryApprovalStore(); + const paused = await AgentExecutor.execute({ agent, input: 'Please call chargeCard now', provider, toolRegistry, approvalStore }); + const failing = { ...provider, generate: async () => { throw new Error('boom Bearer ' + secret); } } as unknown as LLMProvider; + const ended: Span[] = []; + const exporter = { onSpanStart: () => {}, onSpanEnd: (span: Span) => void ended.push({ ...span }) }; + await resumeAfterApproval({ id: paused.approvalId!, approved: true }, approvalStore, toolRegistry, failing, { exporter, ...executeOptions }).catch(() => undefined); + return ended.find((span) => span.name.startsWith('invoke_agent'))!; + }; + + it('scrubs by default, applies a custom redactor, keeps the text under redactContent: false', async () => { + expect(JSON.stringify((await failedRunSpan({})).status)).not.toContain(secret); + expect((await failedRunSpan({ redactContent: (t) => t.replace(secret, '') })).status?.message).toContain(''); + const kept = await failedRunSpan({ redactContent: false }); + expect(kept.status?.message).toContain(secret); + expect(kept.attributes.error).toContain(secret); + }); +}); + describe('Execution - resumeAfterApproval: the span of the approved call (#281)', () => { const approvedRun = async (executeOptions: ResumeExecuteOptions) => { const toolRegistry = new ToolRegistry(); diff --git a/src/execution/resume.ts b/src/execution/resume.ts index 405c8642e..6c6a2f950 100644 --- a/src/execution/resume.ts +++ b/src/execution/resume.ts @@ -54,7 +54,7 @@ import { readonlyPrincipal } from './runPrincipal'; import type { OAuthTokenStore } from '../oauth/types'; import { isSignInRequired, settleSignInRequired, signInOwner, signInRequest, SignInPendingError, type SignInRequired } from '../oauth/signIn'; import { newId } from '../utils/id'; -import { agentRunSpanInit, recordToolOutcome, resolveCaptureContent, toolSpanInit, SUBAGENT_SPAN, type SubagentSpanInfo } from './genAiSpans'; +import { agentRunSpanInit, contentRedactor, recordToolOutcome, resolveCaptureContent, toolSpanInit, SUBAGENT_SPAN, type SubagentSpanInfo } from './genAiSpans'; import { sandboxErrorKind } from '../sandbox/errors'; import { withSpan, type Span } from './tracing'; @@ -242,7 +242,7 @@ function continueDecided(resume: DecidedResume): Promise { ); const init = runSpan(); const resumed: ResumedRun = { ctx, claimed, pending, drift, staleCheckpoint, checkpointStore, runSpan, lease }; - return withSpan(executeOptions.exporter, init.name, init.attributes, (span) => continueInRunSpan(resumed, span), executeOptions.parentSpanId, init.kind); + return withSpan(executeOptions.exporter, init.name, init.attributes, (span) => continueInRunSpan(resumed, span), executeOptions.parentSpanId, init.kind, contentRedactor(executeOptions)); } /** @@ -1124,7 +1124,8 @@ async function runApprovedInSpan(ctx: ResumeContext, pending: PendingApproval, s } }, ctx.runSpanId, - init.kind + init.kind, + contentRedactor({ redactContent }) ); if ('signIn' in outcome) throw outcome.signIn; return outcome.message; diff --git a/src/execution/scrubSecrets.test.ts b/src/execution/scrubSecrets.test.ts index 082ee2479..b4e4c374d 100644 --- a/src/execution/scrubSecrets.test.ts +++ b/src/execution/scrubSecrets.test.ts @@ -1,7 +1,67 @@ import { describe, expect, it } from 'vitest'; import { scrubSecrets } from './scrubSecrets'; +// Key-shaped fixtures are built at runtime so no secret-looking literal ships in source. +const stripe = (kind: string) => ['s', 'k_', kind, '_'].join('') + 'x'.repeat(24); +const stripeRestricted = 'r' + 'k_' + 'live' + '_' + 'y'.repeat(24); +const slack = (kind: string) => 'xox' + kind + '-' + '1234567890-abcdefghij'; +const google = 'AI' + 'za' + 'S'.repeat(35); +const jwt = ['eyJ' + 'hbGciOiJIUzI1', 'eyJ' + 'zdWIiOiIxMjM0', 'SflKxwRJSMeKKF2QT4fw'].join('.'); + describe('scrubSecrets', () => { + it.each([ + ['Stripe live secret', stripe('live')], + ['Stripe test secret', stripe('test')], + ['Stripe restricted', stripeRestricted], + ['Slack bot', slack('b')], + ['Slack app', slack('a')], + ['Slack user', slack('p')], + ['Slack refresh', slack('r')], + ['Slack legacy', slack('s')], + ['Google API key', google], + ['JWT', jwt], + ])('replaces a %s key with [redacted]', (_name, key) => { + expect(scrubSecrets(`value ${key} end`)).toBe('value [redacted] end'); + }); + + it('matches Bearer in any case, keeping the word as written', () => { + const token = 'a'.repeat(24); + expect(scrubSecrets(`bearer ${token}`)).toBe('bearer [redacted]'); + expect(scrubSecrets(`BEARER ${token}`)).toBe('BEARER [redacted]'); + }); + + it('leaves look-alikes of the wider shapes alone', () => { + const text = + 'skill_level_expert_0123456789, xoxo-hugs, xoxb-short, bearer of news, eyJ, eyJhbGciOiJIUzI1NiJ9, AIza_short, sk_live_short'; + expect(scrubSecrets(text)).toBe(text); + }); + + it.each([ + ['repeated Stripe prefixes', 'sk_live_'.repeat(50_000)], + ['a long Stripe run', 'sk_live_' + 'a'.repeat(300_000)], + ['repeated Slack prefixes', 'xoxb-'.repeat(80_000)], + ['repeated Google prefixes', 'AIza'.repeat(80_000)], + ['repeated JWT headers', 'eyJabcdefgh.'.repeat(40_000)], + ['repeated JWT header and payload', 'eyJabcdefgh.eyJabcdefgh'.repeat(20_000)], + ['repeated case-folded Bearer', 'bEaReR '.repeat(50_000)], + // A prefix recurring inside one long run that never completes a match (restart attack). + ['eyJ- restarts inside one token run', 'eyJ-'.repeat(40_000)], + ['eyJ restarts with one dot', 'eyJabcdefgh.eyJ-'.repeat(20_000)], + ['xox restarts in a short run', 'xoxb-short'.repeat(40_000)], + ['sk_ restarts in a short run', 'sk_live_short'.repeat(40_000)], + ['sk- restarts in a short run', 'sk-short'.repeat(40_000)], + ['lsv2_ restarts in a short run', 'lsv2_short'.repeat(40_000)], + ['ghp_ restarts in a short run', 'ghp_short'.repeat(40_000)], + ['github_pat_ restarts in a short run', 'github_pat_short'.repeat(30_000)], + ['AIza restarts in a short run', 'AIzaShort-'.repeat(40_000)], + ['Bearer then long whitespace', 'Bearer' + ' '.repeat(200_000)], + ['Bearer restarts with whitespace', 'Bearer short '.repeat(30_000)], + ])('scrubs %s in linear time', (_name, text) => { + const start = performance.now(); + scrubSecrets(text); + expect(performance.now() - start).toBeLessThan(200); + }); + it.each([ ['OpenRouter', 'sk-or-v1-0123456789abcdef0123456789abcdef'], ['OpenAI project', 'sk-proj-Ab3dEf6hIj9kLm2nOp5qRs8t'], diff --git a/src/execution/scrubSecrets.ts b/src/execution/scrubSecrets.ts index 88cffc2d0..24c338f04 100644 --- a/src/execution/scrubSecrets.ts +++ b/src/execution/scrubSecrets.ts @@ -6,7 +6,7 @@ */ /** What a matched secret is replaced with. */ -export const REDACTED = '[redacted]'; +const REDACTED = '[redacted]'; /** * OpenAI / OpenRouter (`sk-or-v1-`) / Anthropic (`sk-ant-`) style keys. A key @@ -24,10 +24,19 @@ const SECRET_PATTERNS: RegExp[] = [ /\bgithub_pat_[A-Za-z0-9_]{16,}/g, // AWS access key ids. /\bAKIA[0-9A-Z]{16}\b/g, + // Stripe secret / restricted keys. + /\b[sr]k_(?:live|test)_[A-Za-z0-9]{16,}/g, + // Slack bot / app / user / refresh / legacy tokens. + /\bxox[abprs]-[A-Za-z0-9-]{10,}/g, + // Google API keys. + /\bAIza[0-9A-Za-z_-]{35}/g, + // JWTs: base64url header and payload (both `eyJ`, a `{"` object) and a signature. + // Starts only at the head of a token run: a `\b` start also matched inside `eyJ-eyJ-…`, rescanning the run from each `eyJ` (quadratic). + /(?`: the word is kept, the token is redacted. */ -const BEARER = /\b(Bearer\s+)[A-Za-z0-9._~+/=-]{16,}/g; +const BEARER = /\b(Bearer\s+)[A-Za-z0-9._~+/=-]{16,}/gi; /** Replaces API-key shapes in `text` with `[redacted]`. */ export function scrubSecrets(text: string): string { @@ -37,3 +46,14 @@ export function scrubSecrets(text: string): string { } return scrubbed; } + +/** A custom redactor that throws redacts that text whole: telemetry never fails a run. */ +export function safeRedactor(redact: (text: string) => string): (text: string) => string { + return (text) => { + try { + return redact(text); + } catch { + return REDACTED; + } + }; +} diff --git a/src/execution/tracing.test.ts b/src/execution/tracing.test.ts index dc18573c1..866da3655 100644 --- a/src/execution/tracing.test.ts +++ b/src/execution/tracing.test.ts @@ -1,5 +1,56 @@ import { describe, it, expect, vi } from 'vitest'; -import { withSpan, Span, TraceExporter } from './tracing'; +import { withSpan, recordSpanError, Span, TraceExporter } from './tracing'; + +describe('recordSpanError redaction', () => { + const token = 'ghp_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4'; + const fresh = (): Span => ({ id: 's', name: 'x', attributes: {}, startTime: 0 }); + + it('scrubs attrs.error and status.message by default, keeping error.type', () => { + const span = fresh(); + recordSpanError(span, new TypeError(`denied for ${token}`)); + expect(span.attributes.error).toBe('denied for [redacted]'); + expect(span.status).toEqual({ code: 'error', message: 'denied for [redacted]' }); + expect(span.attributes['error.type']).toBe('TypeError'); + }); + + it('keeps the text with an identity redactor and applies a custom one', () => { + const kept = fresh(); + recordSpanError(kept, new Error(`x ${token}`), (text) => text); + expect(kept.status?.message).toBe(`x ${token}`); + const custom = fresh(); + recordSpanError(custom, new Error('Paris'), (text) => text.toUpperCase()); + expect(custom.attributes.error).toBe('PARIS'); + expect(custom.status?.message).toBe('PARIS'); + }); + + it('scrubs a thrown string and leaves a non-string error value as it was', () => { + const text = fresh(); + recordSpanError(text, `bad ${token}`); + expect(text.attributes.error).toBe('bad [redacted]'); + const obj = fresh(); + const value = { code: 7 }; + recordSpanError(obj, value); + expect(obj.attributes.error).toBe(value); + }); + + it('a throwing redactor never replaces the run error', () => { + const span = fresh(); + recordSpanError(span, new Error('real failure'), () => { + throw new Error('redactor bug'); + }); + expect(span.attributes.error).toBe('[redacted]'); + expect(span.status).toEqual({ code: 'error', message: '[redacted]' }); + }); + + it('withSpan passes its redactor to the failure it records', async () => { + const ends: Span[] = []; + const exporter: TraceExporter = { onSpanStart: () => undefined, onSpanEnd: (span) => ends.push({ ...span }) }; + await expect( + withSpan(exporter, 's', {}, async () => { throw new Error(`oops ${token}`); }, undefined, undefined, (text) => text.replace(token, '')) + ).rejects.toThrow('oops'); + expect(ends[0].status?.message).toBe('oops '); + }); +}); function createSpyExporter() { const starts: Span[] = []; diff --git a/src/execution/tracing.ts b/src/execution/tracing.ts index 6e19fbb3f..b5475f809 100644 --- a/src/execution/tracing.ts +++ b/src/execution/tracing.ts @@ -10,6 +10,7 @@ */ import { newId } from '../utils/id'; +import { safeRedactor, scrubSecrets } from './scrubSecrets'; import { ErrorAttr, GenAiAttr, GenAiOperation, LegacyAttr, SdkAttr } from './semconv'; /** OpenTelemetry span kind, for the kinds this SDK emits. */ @@ -41,14 +42,18 @@ export interface Span { /** * Records a failure on `span`: `error.type` (a low-cardinality class, the * error's name), the legacy `error` message attribute, and an error status. + * The message text goes through `redact` (default: the built-in secret + * scrubber) like captured content; `error.type` is never redacted. */ -export function recordSpanError(span: Span, error: unknown): void { +export function recordSpanError(span: Span, error: unknown, redact: (text: string) => string = scrubSecrets): void { + const scrub = safeRedactor(redact); + const message = error instanceof Error ? error.message : error; span.attributes = { ...span.attributes, - [LegacyAttr.ERROR]: error instanceof Error ? error.message : error, + [LegacyAttr.ERROR]: typeof message === 'string' ? scrub(message) : message, [ErrorAttr.TYPE]: error instanceof Error ? error.name : ErrorAttr.OTHER, }; - span.status = { code: 'error', message: error instanceof Error ? error.message : String(error) }; + span.status = { code: 'error', message: scrub(error instanceof Error ? error.message : String(error)) }; } /** Cost and estimation of the finished child spans of one open span. */ @@ -192,6 +197,8 @@ export interface TraceExporter { * - `fn` receives the span so callers can read its generated `id` (e.g. * to pass as the `parentId` of a further-nested withSpan() call). * - `kind` is the OpenTelemetry span kind (default `internal`). + * - `redact` maps the recorded error text (default: the built-in secret + * scrubber); a run passes its `redactContent` choice. * - Before `onSpanEnd`, a span whose child spans carried `lousho.cost_usd` * gets the sum as its own (cumulative) `lousho.cost_usd`, absent when a * child with token usage had no known price, and `lousho.usage.estimated` @@ -204,7 +211,8 @@ export async function withSpan( attributes: Record, fn: (span: Span) => Promise, parentId?: string, - kind?: SpanKind + kind?: SpanKind, + redact?: (text: string) => string ): Promise { const span: Span = { id: newId(), @@ -222,7 +230,7 @@ export async function withSpan( try { return await fn(span); } catch (error) { - recordSpanError(span, error); + recordSpanError(span, error, redact); throw error; } finally { span.endTime = Date.now(); diff --git a/src/flows/FlowExecutor.ts b/src/flows/FlowExecutor.ts index 7e5845b61..619bf84bc 100644 --- a/src/flows/FlowExecutor.ts +++ b/src/flows/FlowExecutor.ts @@ -29,6 +29,7 @@ import { evaluateSafeExpression, ExpressionError } from './safeExpression'; import { isCreateAgentResult } from './validators'; import { Span, TraceExporter, recordSpanError, withSpan } from '../execution/tracing'; import { + contentRedactor, defined, llmSpanInit, recordLlmResult, @@ -622,11 +623,12 @@ export class FlowExecutor { }), async (flowSpan) => { const result = await this.runFlow(flow, { ...context, parentSpanId: flowSpan.id }, onEvent, run); - this.recordOutcome(flowSpan, result.error); + this.recordOutcome(flowSpan, result.error, context); return result; }, context.parentSpanId, - 'internal' + 'internal', + contentRedactor(context) ); } @@ -649,10 +651,10 @@ export class FlowExecutor { } /** Records a flow/node outcome (and the error, when it failed) on its span. */ - private static recordOutcome(span: Span, error?: unknown): void { + private static recordOutcome(span: Span, error: unknown, context: FlowExecutionContext): void { span.attributes = { ...span.attributes, [FlowAttr.OUTCOME]: error ? 'error' : 'success' }; if (error) { - recordSpanError(span, error); + recordSpanError(span, error, contentRedactor(context)); } } @@ -847,15 +849,16 @@ export class FlowExecutor { async (nodeSpan) => { try { const result = await this.runNode(node, stepId, { ...context, parentSpanId: nodeSpan.id }, events, onEvent); - this.recordOutcome(nodeSpan); + this.recordOutcome(nodeSpan, undefined, context); return result; } catch (error) { - this.recordOutcome(nodeSpan, error instanceof FlowApprovalPause ? undefined : error); + this.recordOutcome(nodeSpan, error instanceof FlowApprovalPause ? undefined : error, context); throw error; } }, context.parentSpanId, - 'internal' + 'internal', + contentRedactor(context) ); } @@ -1291,7 +1294,8 @@ export class FlowExecutor { return generated; }, context.parentSpanId, - init.kind + init.kind, + contentRedactor(context) ); // Emit LLM response event @@ -1398,7 +1402,8 @@ export class FlowExecutor { } }, context.parentSpanId, - init.kind + init.kind, + contentRedactor(context) ); }