test(observe): make the loopback redaction test falsifiable; clarify redactContent docs

docs/observability.md

@@ -236,6 +236,19 @@ Prompts, model output and tool arguments/results are sensitive and large, so
 the `gen_ai.*` content attributes are **never recorded by default**. Opt in
 per agent with `createAgent({ exporter, captureContent: true })`.
 
+The exception is a local collector: when `LOUSHO_OBSERVE` points at loopback
+(`1`/`true`, or a URL whose host is `127.0.0.1`, `localhost` or `::1`),
+`createAgent` defaults `captureContent` to `true`, so the daemon's trace view
+shows the prompt, the model output and the tool arguments and results that
+explain a failure. The data stays in the daemon's db on your machine. Turn it
+off with `captureContent: false`; a collector beyond loopback keeps the
+opt-in default. The default applies to agents built with `createAgent`.
+
+`redactContent` keeps only the legacy `input`/`prompt`/`args`/`result`
+attributes off; it does not scrub `gen_ai.*` message content. On a loopback
+collector the message text, including any secret typed into a prompt, is
+stored in the local db, so set `captureContent: false` if that is not wanted.
+
 #### Advanced: the executor API
 
 With the executor, opt in per run:

llms-full.txt

@@ -13896,6 +13896,19 @@ Prompts, model output and tool arguments/results are sensitive and large, so
 the `gen_ai.*` content attributes are **never recorded by default**. Opt in
 per agent with `createAgent({ exporter, captureContent: true })`.
 
+The exception is a local collector: when `LOUSHO_OBSERVE` points at loopback
+(`1`/`true`, or a URL whose host is `127.0.0.1`, `localhost` or `::1`),
+`createAgent` defaults `captureContent` to `true`, so the daemon's trace view
+shows the prompt, the model output and the tool arguments and results that
+explain a failure. The data stays in the daemon's db on your machine. Turn it
+off with `captureContent: false`; a collector beyond loopback keeps the
+opt-in default. The default applies to agents built with `createAgent`.
+
+`redactContent` keeps only the legacy `input`/`prompt`/`args`/`result`
+attributes off; it does not scrub `gen_ai.*` message content. On a loopback
+collector the message text, including any secret typed into a prompt, is
+stored in the local db, so set `captureContent: false` if that is not wanted.
+
 #### Advanced: the executor API
 
 With the executor, opt in per run:

src/createAgent.ts

@@ -20,6 +20,7 @@ import { AgentExecutor, ExecuteOptions, ExecutionResult } from './execution/Agen
 import { streamResumed, throwingRun, type AgentRun } from './execution/agentRun';
 import type { AgentEvent } from './execution/agentEvents';
 import type { TraceExporter } from './execution/tracing';
+import { defaultCaptureContent } from './observe/emit';
 import { withObserveExporter, type PromptUseProvider } from './observe/reporter';
 import { parsePromptRef, registryPrompt } from './observe/promptClient';
 import { assertModelSettings, mergeModelSettings } from './execution/modelSettings';
@@ -409,8 +410,9 @@ export interface CreateAgentBase<TOutput extends StandardSchemaV1 = StandardSche
   /**
    * Record message and tool-argument content on the spans (the `gen_ai.*`
    * content attributes). Off by default because content is sensitive; when
-   * omitted it follows `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`.
-   * Only matters with an `exporter`. See `ExecuteOptions.captureContent`.
+   * omitted it follows `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`,
+   * and is on when `LOUSHO_OBSERVE` points at a loopback collector (set `false`
+   * to opt out). Only matters with an `exporter`. See `ExecuteOptions.captureContent`.
    */
   captureContent?: boolean;
   /**
@@ -1070,9 +1072,12 @@ export function createAgent<TOutput extends StandardSchemaV1 = StandardSchemaV1>
     agentName,
     resolvedInstructions.promptUse !== undefined ? { promptUse: resolvedInstructions.promptUse } : {}
   );
+  // Content defaults on when the env collector is this machine's own (the trace
+  // view cannot explain a failure without it); an explicit setting always wins.
+  const captureContent = defaultCaptureContent(config.captureContent);
   const tracing: Pick<ExecuteOptions, 'exporter' | 'captureContent' | 'redactContent'> = {
     ...(exporter && { exporter }),
-    ...(config.captureContent !== undefined && { captureContent: config.captureContent }),
+    ...(captureContent !== undefined && { captureContent }),
     ...(config.redactContent !== undefined && { redactContent: config.redactContent }),
   };
   const checkpoints = config.store?.checkpoints;

src/createAgentTracing.test.ts

@@ -6,7 +6,7 @@ import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import type { Tracer } from '@opentelemetry/api';
-import { describe, expect, it } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 import { z } from 'zod';
 import { createAgent } from './createAgent';
 import { PropagatingToolError } from './execution/AgentExecutor';
@@ -331,3 +331,78 @@ describe('createAgent({ exporter })', () => {
     });
   });
 });
+
+describe('captureContent default on a local collector', () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.unstubAllGlobals();
+  });
+  const observe = (value: string) => {
+    vi.stubEnv('LOUSHO_OBSERVE', value);
+    vi.stubGlobal('fetch', async () => new Response('{}', { status: 200 }));
+  };
+  const chatInput = async (options: { captureContent?: boolean; redactContent?: boolean } = {}) => {
+    const exporter = recording();
+    await createAgent({ provider: mockModel(['Hi.']), exporter, ...options }).send('my key is sk-secret123');
+    return exporter.ended.find((span) => op(span) === 'chat')!;
+  };
+
+  it('captures content by default when LOUSHO_OBSERVE points at loopback', async () => {
+    observe('1');
+    const chat = await chatInput();
+    expect(chat.attributes['gen_ai.input.messages']).toContain('my key is');
+    expect(chat.attributes['gen_ai.output.messages']).toContain('Hi.');
+  });
+
+  it('does not capture by default for a non-loopback collector', async () => {
+    observe('http://collector.test:7777');
+    expect((await chatInput()).attributes['gen_ai.input.messages']).toBeUndefined();
+  });
+
+  it('does not capture by default when LOUSHO_OBSERVE is unset', async () => {
+    vi.stubEnv('LOUSHO_OBSERVE', '');
+    expect((await chatInput()).attributes['gen_ai.input.messages']).toBeUndefined();
+  });
+
+  it('captureContent: false wins over the loopback default', async () => {
+    observe('1');
+    expect((await chatInput({ captureContent: false })).attributes['gen_ai.input.messages']).toBeUndefined();
+  });
+
+  it('redactContent: false records the deprecated prompt next to the loopback-captured content', async () => {
+    observe('1');
+    const chat = await chatInput({ redactContent: false });
+    expect(chat.attributes['gen_ai.input.messages']).toContain('my key is');
+    expect(chat.attributes.prompt).toBeDefined();
+  });
+
+  it('redactContent: true keeps the deprecated prompt off while gen_ai content is still captured', async () => {
+    observe('1');
+    const chat = await chatInput({ redactContent: true });
+    expect(chat.attributes['gen_ai.input.messages']).toContain('my key is');
+    expect(chat.attributes.prompt).toBeUndefined();
+  });
+
+  it('resume() captures content by default on loopback', async () => {
+    observe('1');
+    const store = memoryStore();
+    let crashed = false;
+    const flaky = defineTool({
+      name: 'flaky',
+      description: 'Dies once',
+      input: z.object({}),
+      execute: async () => {
+        if (!crashed) {
+          crashed = true;
+          throw new PropagatingToolError('process died');
+        }
+        return 'ok';
+      },
+    });
+    const crashing = createAgent({ provider: mockModel([{ toolCalls: [{ name: 'flaky', id: 'c1' }] }]), tools: [flaky], store });
+    await expect(crashing.send('Go', { sessionId: 'job' })).rejects.toThrow('process died');
+    const exporter = recording();
+    await createAgent({ provider: mockModel(['Done.']), tools: [flaky], store, exporter }).resume('job');
+    expect(exporter.ended.find((span) => op(span) === 'chat')!.attributes['gen_ai.output.messages']).toContain('Done.');
+  });
+});

src/observe/emit.test.ts

@@ -5,7 +5,7 @@
  */
 import { describe, expect, it, vi } from 'vitest';
 import type { IngestEvent } from './envelope';
-import { emitObserveEvent, emitObserveEvents, observeTargetFromEnv, resetObserveEnvWarning } from './emit';
+import { emitObserveEvent, emitObserveEvents, isLoopbackTarget, observeTargetFromEnv, resetObserveEnvWarning } from './emit';
 
 const score = (id: string): IngestEvent => ({
   type: 'score',
@@ -112,3 +112,12 @@ describe('emitObserveEvents', () => {
     }
   });
 });
+
+describe('isLoopbackTarget', () => {
+  it.each(['http://127.0.0.1:7777', 'http://localhost:1234', 'http://[::1]:7777', 'https://LOCALHOST'])('%s is loopback', (url) => {
+    expect(isLoopbackTarget({ url })).toBe(true);
+  });
+  it.each(['http://collector.test:7777', 'https://observe.example.com', 'http://0.0.0.0:7777', 'http://localhost.evil.com', 'not a url'])('%s is not', (url) => {
+    expect(isLoopbackTarget({ url })).toBe(false);
+  });
+});

src/observe/emit.ts

@@ -76,6 +76,31 @@ export function observeTargetFromEnv(env: NodeJS.ProcessEnv | undefined = typeof
   };
 }
 
+/**
+ * Whether `target` is this machine's own collector (host `127.0.0.1`,
+ * `localhost` or `::1`). A loopback collector is auth-free and its db never
+ * leaves the machine, so content capture defaults on for it.
+ */
+export function isLoopbackTarget(target: ObserveTarget): boolean {
+  try {
+    const host = new URL(target.url).hostname.toLowerCase();
+    return host === '127.0.0.1' || host === 'localhost' || host === '[::1]';
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * The `captureContent` an agent runs with: the explicit setting when there is
+ * one, else on when the env collector is loopback, else `undefined` (the
+ * executor then follows the OTel content env var).
+ */
+export function defaultCaptureContent(explicit: boolean | undefined, env?: NodeJS.ProcessEnv): boolean | undefined {
+  if (explicit !== undefined) return explicit;
+  const target = observeTargetFromEnv(env);
+  return target && isLoopbackTarget(target) ? true : undefined;
+}
+
 export interface EmitObserveOptions {
   /** Explicit target; default the env's (`observeTargetFromEnv`). */
   target?: ObserveTarget;