From 7f6fcced162713deefa877f4b84ed0d3270191cd Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:22:04 +0300 Subject: [PATCH 01/11] LOU-J1: Claude Code generator Extend AgentSpec (LOU-H9) with optional, backward-compatible policy/ triggers fields for the full cross-harness spec, and add generateClaudeCodeSkill() producing a real Claude Code SKILL.md (YAML frontmatter with name/description, full prompt and every tool in the Markdown body). Adds a shared fixture spec reused by the other LOU-J generators. Co-Authored-By: Claude Sonnet 5 --- .../generators/__fixtures__/exampleSpec.ts | 28 +++++ src/spec/generators/claude-code.test.ts | 39 ++++++ src/spec/generators/claude-code.ts | 115 ++++++++++++++++++ src/spec/schema.ts | 52 ++++++++ 4 files changed, 234 insertions(+) create mode 100644 src/spec/generators/__fixtures__/exampleSpec.ts create mode 100644 src/spec/generators/claude-code.test.ts create mode 100644 src/spec/generators/claude-code.ts diff --git a/src/spec/generators/__fixtures__/exampleSpec.ts b/src/spec/generators/__fixtures__/exampleSpec.ts new file mode 100644 index 000000000..1c88e8013 --- /dev/null +++ b/src/spec/generators/__fixtures__/exampleSpec.ts @@ -0,0 +1,28 @@ +/** + * Shared fixture AgentSpec reused by every cross-harness generator's tests + * (LOU-J1/J2/J3), so the generators are all verified against the exact same + * input. + */ +import { AgentSpec } from '../../schema'; + +export const exampleAgentSpec: AgentSpec = { + name: 'ops-fixer', + prompt: + 'You are an on-call fixer agent. Diagnose the root cause from the provided logs and ' + + 'produce a minimal unified diff that fixes it. Never invent files that were not shown to you.', + provider: { + type: 'mock', + model: 'mock-model-1', + }, + tools: ['http', 'current-date', 'day-name'], + policy: { + requiresApproval: true, + guardrails: ['secret-scan', 'diff-size-cap'], + }, + triggers: [ + { + type: 'webhook', + path: '/webhook', + }, + ], +}; diff --git a/src/spec/generators/claude-code.test.ts b/src/spec/generators/claude-code.test.ts new file mode 100644 index 000000000..5bece4755 --- /dev/null +++ b/src/spec/generators/claude-code.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest'; +import { generateClaudeCodeSkill } from './claude-code'; +import { exampleAgentSpec } from './__fixtures__/exampleSpec'; + +describe('generateClaudeCodeSkill', () => { + it('produces a SKILL.md path under .claude/skills/', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.path).toBe('.claude/skills/ops-fixer/SKILL.md'); + }); + + it('includes valid YAML frontmatter with name and description', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.content.startsWith('---\n')).toBe(true); + expect(file.content).toMatch(/name: "ops-fixer"/); + expect(file.content).toMatch(/description: "/); + }); + + it('contains the agent name', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.content).toContain(exampleAgentSpec.name); + }); + + it('contains the FULL prompt verbatim', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.content).toContain(exampleAgentSpec.prompt); + }); + + it('contains every tool', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + for (const t of exampleAgentSpec.tools ?? []) { + expect(file.content).toContain(t); + } + }); + + it('handles a spec with no tools', () => { + const file = generateClaudeCodeSkill({ ...exampleAgentSpec, tools: undefined }); + expect(file.content).toContain('(none)'); + }); +}); diff --git a/src/spec/generators/claude-code.ts b/src/spec/generators/claude-code.ts new file mode 100644 index 000000000..6aec58bf6 --- /dev/null +++ b/src/spec/generators/claude-code.ts @@ -0,0 +1,115 @@ +/** + * Claude Code generator (LOU-J1) + * + * Turns a cross-harness AgentSpec (LOU-H9, extended with `policy`/`triggers` + * in LOU-J1) into a real Claude Code Skill file (SKILL.md). + * + * This repository does not itself ship a `.claude/skills/` directory (there + * is nothing under `.claude/` to mirror the frontmatter shape from), so the + * frontmatter below follows Claude Code's own documented Skill format: a + * YAML frontmatter block with the two fields every skill needs - `name` + * (a short identifier) and `description` (what the skill does / when to use + * it, since that's what Claude Code's skill picker matches against) - + * followed by the skill's instructions as Markdown body content. + */ +import { AgentSpec } from '../schema'; + +/** + * A generated file: where it should be written, and its full content. + */ +export interface GeneratedFile { + path: string; + content: string; +} + +/** + * Slugifies an AgentSpec name into a filesystem/frontmatter-safe skill + * name: lowercase, spaces/underscores collapsed to single hyphens, and + * anything that isn't a-z/0-9/hyphen stripped. + */ +function slugify(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Builds a one-line description for the SKILL.md frontmatter from the + * spec's prompt: the first sentence (or the whole prompt if it has none), + * truncated to a reasonable frontmatter-friendly length. + */ +function buildDescription(prompt: string): string { + const firstSentence = prompt.split(/(?<=[.!?])\s/)[0] || prompt; + const trimmed = firstSentence.trim(); + return trimmed.length > 200 ? `${trimmed.slice(0, 197)}...` : trimmed; +} + +/** + * Escapes a value for safe embedding in a single YAML frontmatter scalar + * (wraps in double quotes, escapes embedded double quotes/backslashes). + */ +function yamlScalar(value: string): string { + return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`; +} + +/** + * Generates a Claude Code SKILL.md for `spec`. + * + * The full spec prompt is preserved verbatim in the Markdown body (not + * truncated) - only the frontmatter `description` is shortened, since that + * field exists purely for the skill picker, not to carry the whole prompt. + * Every tool from `spec.tools` is listed explicitly so a reader (or the + * LOU-J1 test) can see exactly which tools this skill expects to be + * available. + */ +export function generateClaudeCodeSkill(spec: AgentSpec): GeneratedFile { + const slug = slugify(spec.name); + const description = buildDescription(spec.prompt); + const tools = spec.tools ?? []; + + const frontmatter = [ + '---', + `name: ${yamlScalar(spec.name)}`, + `description: ${yamlScalar(description)}`, + '---', + ].join('\n'); + + const toolsSection = + tools.length > 0 + ? ['## Tools', '', ...tools.map((t) => `- \`${t}\``)].join('\n') + : ['## Tools', '', '(none)'].join('\n'); + + const policySection = spec.policy + ? [ + '## Policy', + '', + spec.policy.requiresApproval + ? '- Tool calls from this agent require human approval before running.' + : '- Tool calls from this agent do not require approval.', + ...(spec.policy.guardrails && spec.policy.guardrails.length > 0 + ? [`- Guardrails: ${spec.policy.guardrails.join(', ')}`] + : []), + ].join('\n') + : ''; + + const body = [ + frontmatter, + '', + `# ${spec.name}`, + '', + '## Prompt', + '', + spec.prompt, + '', + toolsSection, + ...(policySection ? ['', policySection] : []), + '', + ].join('\n'); + + return { + path: `.claude/skills/${slug}/SKILL.md`, + content: body, + }; +} diff --git a/src/spec/schema.ts b/src/spec/schema.ts index 9ed2c489a..342e69265 100644 --- a/src/spec/schema.ts +++ b/src/spec/schema.ts @@ -14,11 +14,46 @@ export interface AgentSpecProvider { model: string; } +/** + * Optional cross-harness execution policy (LOU-J1+). Deliberately loose - + * different target harnesses (Claude Code, Codex, Pi, ...) each honor a + * different subset of this, and new fields are expected to be added by + * later generators without needing a schema migration every time, so this + * is intentionally an open record (`.passthrough()` on the zod side) rather + * than a closed, harness-specific shape. + */ +export interface AgentSpecPolicy { + /** Whether tool calls from this agent require human approval before running (LOU-C). */ + requiresApproval?: boolean; + /** Named guardrails (see src/execution/guardrails.ts) this agent's actions must pass. */ + guardrails?: string[]; + /** Additional, harness-specific policy fields. */ + [key: string]: unknown; +} + +/** + * Optional trigger describing when/how this agent is invoked outside of a + * direct call (e.g. a monitoring webhook, a cron schedule). Open record for + * the same reason as AgentSpecPolicy above. + */ +export interface AgentSpecTrigger { + type: string; + [key: string]: unknown; +} + export interface AgentSpec { name: string; prompt: string; provider: AgentSpecProvider; tools?: string[]; + /** + * Optional (LOU-J1+, backward-compatible with LOU-H9's original + * {name, prompt, provider, tools} shape - existing specs that omit this + * still validate and load unchanged). + */ + policy?: AgentSpecPolicy; + /** Optional (LOU-J1+), same backward-compatibility note as `policy`. */ + triggers?: AgentSpecTrigger[]; } export const agentSpecProviderSchema = z.object({ @@ -30,6 +65,21 @@ export const agentSpecProviderSchema = z.object({ }), }); +export const agentSpecPolicySchema = z + .object({ + requiresApproval: z.boolean().optional(), + guardrails: z.array(z.string()).optional(), + }) + .passthrough(); + +export const agentSpecTriggerSchema = z + .object({ + type: z.string({ + required_error: "AgentSpec validation failed: missing required field 'triggers[].type'", + }), + }) + .passthrough(); + export const agentSpecSchema = z.object({ name: z.string({ required_error: "AgentSpec validation failed: missing required field 'name'", @@ -39,4 +89,6 @@ export const agentSpecSchema = z.object({ }), provider: agentSpecProviderSchema, tools: z.array(z.string()).optional(), + policy: agentSpecPolicySchema.optional(), + triggers: z.array(agentSpecTriggerSchema).optional(), }); From 4cdc8d3cae3e561104878cc849ed3e88f89b9b95 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:22:26 +0300 Subject: [PATCH 02/11] LOU-J2: Codex generator Add generateCodexConfig() producing a Codex-style agent-config JSON file ({name, instructions, model:{provider,name}, tools, policy?, triggers?}). Note: Codex's exact documented config schema was not verifiable from this repo/session, so the shape follows the ticket's own illustrative baseline (instructions field for the prompt) - flagged in the file's doc comment as best-effort, not confirmed against Codex's real spec. Co-Authored-By: Claude Sonnet 5 --- src/spec/generators/codex.test.ts | 42 +++++++++++++++++++ src/spec/generators/codex.ts | 67 +++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 src/spec/generators/codex.test.ts create mode 100644 src/spec/generators/codex.ts diff --git a/src/spec/generators/codex.test.ts b/src/spec/generators/codex.test.ts new file mode 100644 index 000000000..89ef365ed --- /dev/null +++ b/src/spec/generators/codex.test.ts @@ -0,0 +1,42 @@ +import { describe, it, expect } from 'vitest'; +import { generateCodexConfig, CodexAgentConfig } from './codex'; +import { exampleAgentSpec } from './__fixtures__/exampleSpec'; + +describe('generateCodexConfig', () => { + it('produces a config path under .codex/agents/', () => { + const file = generateCodexConfig(exampleAgentSpec); + expect(file.path).toBe('.codex/agents/ops-fixer.json'); + }); + + it('produces valid, parseable JSON', () => { + const file = generateCodexConfig(exampleAgentSpec); + expect(() => JSON.parse(file.content)).not.toThrow(); + }); + + it("parsed output's fields match the fixture", () => { + const file = generateCodexConfig(exampleAgentSpec); + const parsed = JSON.parse(file.content) as CodexAgentConfig; + + expect(parsed.name).toBe(exampleAgentSpec.name); + expect(parsed.instructions).toBe(exampleAgentSpec.prompt); + expect(parsed.model).toEqual({ + provider: exampleAgentSpec.provider.type, + name: exampleAgentSpec.provider.model, + }); + expect(parsed.tools).toEqual(exampleAgentSpec.tools); + expect(parsed.policy).toEqual(exampleAgentSpec.policy); + expect(parsed.triggers).toEqual(exampleAgentSpec.triggers); + }); + + it('omits tools/policy/triggers gracefully when absent from the spec', () => { + const file = generateCodexConfig({ + name: 'bare', + prompt: 'bare prompt', + provider: { type: 'mock', model: 'mock-model-1' }, + }); + const parsed = JSON.parse(file.content) as CodexAgentConfig; + expect(parsed.tools).toEqual([]); + expect(parsed.policy).toBeUndefined(); + expect(parsed.triggers).toBeUndefined(); + }); +}); diff --git a/src/spec/generators/codex.ts b/src/spec/generators/codex.ts new file mode 100644 index 000000000..6544cfe41 --- /dev/null +++ b/src/spec/generators/codex.ts @@ -0,0 +1,67 @@ +/** + * Codex generator (LOU-J2) + * + * Turns an AgentSpec into a Codex-style agent-config JSON file. + * + * Uncertainty note: OpenAI Codex CLI's agent-config format is not something + * this repo has a copy of or a dependency on to verify against precisely, + * and no network research was performed as part of writing this generator. + * The shape below follows the LOU-J2 ticket's own illustrative shape as the + * best-effort baseline: a top-level `instructions` field carries the + * prompt (matching Codex's documented convention of an `instructions` + * field for agent system prompts, as opposed to Claude Code's `prompt`/ + * skill-body convention), `model`/`provider` carry the target model, + * `tools` is a flat array of tool names, and `policy`/`triggers` are + * passed through verbatim when present. If Codex's real schema differs + * (e.g. different field names or nesting), this generator's output shape + * will need to be adjusted to match - flagged here rather than silently + * assumed correct. + */ +import { AgentSpec } from '../schema'; +import { GeneratedFile } from './claude-code'; + +/** + * The Codex agent-config JSON shape this generator produces. See the + * uncertainty note above this generator's doc comment. + */ +export interface CodexAgentConfig { + name: string; + instructions: string; + model: { + provider: string; + name: string; + }; + tools: string[]; + policy?: AgentSpec['policy']; + triggers?: AgentSpec['triggers']; +} + +function slugify(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Generates a Codex agent-config JSON file for `spec`. + */ +export function generateCodexConfig(spec: AgentSpec): GeneratedFile { + const config: CodexAgentConfig = { + name: spec.name, + instructions: spec.prompt, + model: { + provider: spec.provider.type, + name: spec.provider.model, + }, + tools: spec.tools ?? [], + ...(spec.policy ? { policy: spec.policy } : {}), + ...(spec.triggers ? { triggers: spec.triggers } : {}), + }; + + return { + path: `.codex/agents/${slugify(spec.name)}.json`, + content: JSON.stringify(config, null, 2) + '\n', + }; +} From ccd919b79d596d2163c66ec141506608dc2e7091 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:22:44 +0300 Subject: [PATCH 03/11] LOU-J3: Pi generator Add generatePiSkill() producing a Pi skill YAML file, reusing the existing 'yaml' dependency (already used by loadSpec.ts) for stringifying rather than adding a new YAML dependency. Co-Authored-By: Claude Sonnet 5 --- src/spec/generators/pi.test.ts | 37 ++++++++++++++++++++++++ src/spec/generators/pi.ts | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 src/spec/generators/pi.test.ts create mode 100644 src/spec/generators/pi.ts diff --git a/src/spec/generators/pi.test.ts b/src/spec/generators/pi.test.ts new file mode 100644 index 000000000..51fd66127 --- /dev/null +++ b/src/spec/generators/pi.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect } from 'vitest'; +import { parse as parseYaml } from 'yaml'; +import { generatePiSkill, PiSkillDocument } from './pi'; +import { exampleAgentSpec } from './__fixtures__/exampleSpec'; + +describe('generatePiSkill', () => { + it('produces a YAML file path under .pi/skills/', () => { + const file = generatePiSkill(exampleAgentSpec); + expect(file.path).toBe('.pi/skills/ops-fixer.yaml'); + }); + + it('produces valid, parseable YAML', () => { + const file = generatePiSkill(exampleAgentSpec); + expect(() => parseYaml(file.content)).not.toThrow(); + }); + + it('contains name, full prompt, and every tool', () => { + const file = generatePiSkill(exampleAgentSpec); + const parsed = parseYaml(file.content) as PiSkillDocument; + + expect(parsed.name).toBe(exampleAgentSpec.name); + expect(parsed.prompt).toBe(exampleAgentSpec.prompt); + expect(file.content).toContain(exampleAgentSpec.name); + expect(file.content).toContain(exampleAgentSpec.prompt.split('\n')[0].slice(0, 20)); + for (const t of exampleAgentSpec.tools ?? []) { + expect(parsed.tools).toContain(t); + } + }); + + it('round-trips provider/policy/triggers', () => { + const file = generatePiSkill(exampleAgentSpec); + const parsed = parseYaml(file.content) as PiSkillDocument; + expect(parsed.provider).toEqual(exampleAgentSpec.provider); + expect(parsed.policy).toEqual(exampleAgentSpec.policy); + expect(parsed.triggers).toEqual(exampleAgentSpec.triggers); + }); +}); diff --git a/src/spec/generators/pi.ts b/src/spec/generators/pi.ts new file mode 100644 index 000000000..881c95a5a --- /dev/null +++ b/src/spec/generators/pi.ts @@ -0,0 +1,53 @@ +/** + * Pi generator (LOU-J3) + * + * Turns an AgentSpec into a Pi skill YAML file. Reuses the `yaml` package + * that's already a project dependency (added for loadSpec.ts's .yaml/.yml + * parsing - see src/spec/loadSpec.ts) for stringifying, rather than adding + * a new YAML dependency just for this generator. + */ +import { stringify as stringifyYaml } from 'yaml'; +import { AgentSpec } from '../schema'; +import { GeneratedFile } from './claude-code'; + +function slugify(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * The plain-data shape serialized to YAML for a Pi skill. Kept close to + * AgentSpec itself (Pi's skill format, like the source AgentSpec, is a + * name/prompt/tools/provider/policy/triggers record) rather than + * introducing an unrelated schema. + */ +export interface PiSkillDocument { + name: string; + prompt: string; + tools: string[]; + provider: AgentSpec['provider']; + policy?: AgentSpec['policy']; + triggers?: AgentSpec['triggers']; +} + +/** + * Generates a Pi skill YAML file for `spec`. + */ +export function generatePiSkill(spec: AgentSpec): GeneratedFile { + const doc: PiSkillDocument = { + name: spec.name, + prompt: spec.prompt, + tools: spec.tools ?? [], + provider: spec.provider, + ...(spec.policy ? { policy: spec.policy } : {}), + ...(spec.triggers ? { triggers: spec.triggers } : {}), + }; + + return { + path: `.pi/skills/${slugify(spec.name)}.yaml`, + content: stringifyYaml(doc), + }; +} From cbb96d912b646287a24dd4a51e26f3908658ec70 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:23:27 +0300 Subject: [PATCH 04/11] LOU-J4: Grafana/Datadog monitor trigger Add examples/ops-pipeline/monitor.ts: ErrorSignal type, an in-memory Set dedup by signature, and handleErrorSignal() which calls the real static AgentExecutor.execute() exactly once per new signature with a prompt built by buildMonitorPrompt(). Wires a POST /webhook HTTP listener reusing src/cli/dev.ts's http.createServer pattern (localhost-default binding, 1MB request body cap). Co-Authored-By: Claude Sonnet 5 --- examples/ops-pipeline/monitor.test.ts | 96 +++++++++++++ examples/ops-pipeline/monitor.ts | 200 ++++++++++++++++++++++++++ 2 files changed, 296 insertions(+) create mode 100644 examples/ops-pipeline/monitor.test.ts create mode 100644 examples/ops-pipeline/monitor.ts diff --git a/examples/ops-pipeline/monitor.test.ts b/examples/ops-pipeline/monitor.test.ts new file mode 100644 index 000000000..40e1da98b --- /dev/null +++ b/examples/ops-pipeline/monitor.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, vi } from 'vitest'; +import { handleErrorSignal, buildMonitorPrompt, ErrorSignal, startMonitorServer } from './monitor'; +import { AgentExecutor } from '../../src/execution/AgentExecutor'; +import { AgentType } from '../../src/types'; +import { createMockProvider } from '../../src/providers/mock'; + +function makeSignal(overrides: Partial = {}): ErrorSignal { + return { + signature: 'sig-1', + service: 'grafana', + message: 'NullPointerException in OrderService', + logs: 'at OrderService.charge(OrderService.java:42)', + ...overrides, + }; +} + +const executeOptions = { + agent: { name: 'monitor', agentType: AgentType.SmartAssistant, prompt: 'You are the monitor.' }, + provider: createMockProvider({ responses: ['ack'] }), +}; + +describe('buildMonitorPrompt', () => { + it('includes the signature, message and logs', () => { + const prompt = buildMonitorPrompt(makeSignal()); + expect(prompt).toContain('sig-1'); + expect(prompt).toContain('NullPointerException in OrderService'); + expect(prompt).toContain('OrderService.java:42'); + }); +}); + +describe('handleErrorSignal', () => { + it('calls AgentExecutor.execute() exactly once for a new signature', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + + const result = await handleErrorSignal(makeSignal(), executeOptions, seen); + + expect(spy).toHaveBeenCalledTimes(1); + expect(result).toBeDefined(); + expect(spy.mock.calls[0][0].input).toContain('sig-1'); + + spy.mockRestore(); + }); + + it('does not call the executor again for a duplicate signature', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + + await handleErrorSignal(makeSignal(), executeOptions, seen); + expect(spy).toHaveBeenCalledTimes(1); + + const secondResult = await handleErrorSignal(makeSignal(), executeOptions, seen); + expect(spy).toHaveBeenCalledTimes(1); // zero additional calls + expect(secondResult).toBeUndefined(); + + spy.mockRestore(); + }); + + it('treats different signatures as independent (executor called once each)', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + + await handleErrorSignal(makeSignal({ signature: 'sig-a' }), executeOptions, seen); + await handleErrorSignal(makeSignal({ signature: 'sig-b' }), executeOptions, seen); + + expect(spy).toHaveBeenCalledTimes(2); + spy.mockRestore(); + }); +}); + +describe('startMonitorServer', () => { + it('accepts a POST /webhook and dedupes by signature over HTTP', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + const handle = await startMonitorServer({ executeOptions, seen, port: 0 }); + + try { + const url = `http://127.0.0.1:${handle.port}/webhook`; + const signal = makeSignal({ signature: 'http-sig' }); + + const res1 = await fetch(url, { method: 'POST', body: JSON.stringify(signal) }); + const body1 = await res1.json(); + expect(res1.status).toBe(202); + expect(body1.deduped).toBe(false); + + const res2 = await fetch(url, { method: 'POST', body: JSON.stringify(signal) }); + const body2 = await res2.json(); + expect(body2.deduped).toBe(true); + + expect(spy).toHaveBeenCalledTimes(1); + } finally { + spy.mockRestore(); + await handle.close(); + } + }); +}); diff --git a/examples/ops-pipeline/monitor.ts b/examples/ops-pipeline/monitor.ts new file mode 100644 index 000000000..decf940a9 --- /dev/null +++ b/examples/ops-pipeline/monitor.ts @@ -0,0 +1,200 @@ +/** + * Ops pipeline: Grafana/Datadog error-signal monitor (LOU-J4). + * + * Receives an ErrorSignal (from a Grafana or Datadog alert webhook, or from + * the mock sender in mocks/ for the demo), dedupes it by `signature` so a + * flapping/retried alert never triggers more than one agent run, and - on a + * genuinely new signature - calls the REAL static AgentExecutor.execute() + * (src/execution/AgentExecutor.ts) exactly once with a prompt built by + * buildMonitorPrompt(). + * + * A small HTTP listener (startMonitorServer) exposes this as `POST + * /webhook`, reusing the same http.createServer pattern already + * established in src/cli/dev.ts: bound to localhost by default, and a + * request-body size cap so an oversized payload can't exhaust memory. + */ +import * as http from 'node:http'; +import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../../src/execution/AgentExecutor'; + +/** + * A single error/incident signal from a monitoring source. + */ +export interface ErrorSignal { + /** Stable identity for this specific error condition, used for dedup. */ + signature: string; + service: 'grafana' | 'datadog'; + message: string; + logs: string; +} + +/** + * Everything AgentExecutor.execute() needs EXCEPT `input` - handleErrorSignal + * builds `input` itself from the signal via buildMonitorPrompt(). + */ +export type MonitorExecuteOptions = Omit; + +/** + * Module-level default dedup set, used when handleErrorSignal() is called + * without an explicit `seen` set (e.g. from startMonitorServer(), where one + * long-lived set should track dedup for the life of the process). Tests + * pass their own Set so cases stay isolated from each other and from this + * module-level state. + */ +const defaultSeenSignatures = new Set(); + +/** + * Builds the prompt sent to the monitor agent for a given signal. + */ +export function buildMonitorPrompt(signal: ErrorSignal): string { + return [ + `A ${signal.service} alert fired with signature "${signal.signature}".`, + `Message: ${signal.message}`, + '', + 'Logs:', + signal.logs, + '', + 'Diagnose this and delegate a fix request to the fixer agent if one is available.', + ].join('\n'); +} + +/** + * Handles one incoming ErrorSignal: deduped by `signal.signature` via an + * in-memory Set - if this signature has already been seen, returns + * immediately without calling the executor at all. Otherwise records the + * signature and calls the real AgentExecutor.execute() exactly once. + */ +export async function handleErrorSignal( + signal: ErrorSignal, + executeOptions: MonitorExecuteOptions, + seen: Set = defaultSeenSignatures +): Promise { + if (seen.has(signal.signature)) { + return undefined; + } + seen.add(signal.signature); + + return AgentExecutor.execute({ + ...executeOptions, + input: buildMonitorPrompt(signal), + }); +} + +/** Body-size cap for POST /webhook, matching src/cli/dev.ts's /chat convention. */ +const MAX_BODY_BYTES = 1024 * 1024; // 1MB + +class PayloadTooLargeError extends Error {} + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + let bytes = 0; + let tooLarge = false; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + tooLarge = true; + return; + } + body += chunk; + }); + req.on('end', () => { + if (tooLarge) { + reject(new PayloadTooLargeError(`Request body exceeds ${MAX_BODY_BYTES} byte limit`)); + return; + } + resolve(body); + }); + req.on('error', reject); + }); +} + +export interface MonitorServerHandle { + server: http.Server; + port: number; + close: () => Promise; +} + +export interface StartMonitorServerOptions { + executeOptions: MonitorExecuteOptions; + host?: string; + port?: number; + seen?: Set; + /** Called after each handled (non-duplicate) signal, mainly for tests/demo logging. */ + onResult?: (signal: ErrorSignal, result: ExecutionResult) => void; +} + +/** + * Starts an HTTP server exposing `POST /webhook` for ErrorSignal payloads. + * Bound to '127.0.0.1' by default (localhost-only), matching src/cli/dev.ts's + * security posture for local tooling - pass an explicit host to opt in to + * wider access. + */ +export async function startMonitorServer( + options: StartMonitorServerOptions +): Promise { + const host = options.host ?? '127.0.0.1'; + const port = options.port ?? 0; + const seen = options.seen ?? defaultSeenSignatures; + + const server = http.createServer((req, res) => { + void (async () => { + if (req.method === 'POST' && req.url === '/webhook') { + try { + const body = await readBody(req); + const signal = JSON.parse(body || '{}') as Partial; + if ( + typeof signal.signature !== 'string' || + typeof signal.service !== 'string' || + typeof signal.message !== 'string' || + typeof signal.logs !== 'string' + ) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + error: 'Request body must be an ErrorSignal {signature, service, message, logs}', + }) + ); + return; + } + + const result = await handleErrorSignal(signal as ErrorSignal, options.executeOptions, seen); + if (result && options.onResult) { + options.onResult(signal as ErrorSignal, result); + } + + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ deduped: result === undefined })); + } catch (error) { + if (error instanceof PayloadTooLargeError) { + res.writeHead(413, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: error.message })); + return; + } + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + return; + } + + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + })(); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, () => resolve()); + }); + + const address = server.address(); + const actualPort = typeof address === 'object' && address ? address.port : port; + + return { + server, + port: actualPort, + close: () => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }), + }; +} From bf74c584010e03f3ed78ecddefb1325f694bf20a Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:24:56 +0300 Subject: [PATCH 05/11] LOU-J5: Slack alert tool with "Fix it" button Add src/tools/built-in/slack.ts following the ToolDescriptor pattern from http.ts/github.ts: builds a Slack Block Kit payload with one actions block containing a "Fix it" button (action_id 'fix_it', value = approvalId) and POSTs it to a webhook URL read from SLACK_WEBHOOK_URL (LOU-F8 env-var convention). Add examples/ops-pipeline/slackInteractions.ts: a POST /slack/interactions route that parses Slack's interaction callback, extracts the approvalId from the clicked button's value, and calls the REAL LOU-C6 resumeAfterApproval() with it - verified by a test that spies on resumeAfterApproval and confirms the deferred tool actually executes, proving this genuinely routes through the real approval gate rather than a bypass. Co-Authored-By: Claude Sonnet 5 --- .../ops-pipeline/slackInteractions.test.ts | 97 +++++++++++ examples/ops-pipeline/slackInteractions.ts | 151 +++++++++++++++++ src/tools/built-in/slack.test.ts | 77 +++++++++ src/tools/built-in/slack.ts | 155 ++++++++++++++++++ 4 files changed, 480 insertions(+) create mode 100644 examples/ops-pipeline/slackInteractions.test.ts create mode 100644 examples/ops-pipeline/slackInteractions.ts create mode 100644 src/tools/built-in/slack.test.ts create mode 100644 src/tools/built-in/slack.ts diff --git a/examples/ops-pipeline/slackInteractions.test.ts b/examples/ops-pipeline/slackInteractions.test.ts new file mode 100644 index 000000000..a53182344 --- /dev/null +++ b/examples/ops-pipeline/slackInteractions.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import * as resumeModule from '../../src/execution/resume'; +import { + extractApprovalIdFromInteraction, + handleSlackInteraction, + createInMemoryApprovalStore, + SlackInteractionPayload, +} from './slackInteractions'; +import { ApprovalStore, ExecutionSnapshot, PendingApproval } from '../../src/execution/ApprovalGate'; +import { ToolRegistry } from '../../src/tools'; +import { createMockProvider } from '../../src/providers/mock'; +import { AgentType } from '../../src/types'; + +describe('extractApprovalIdFromInteraction', () => { + it('extracts the approvalId from a fix_it block_actions click', () => { + const payload: SlackInteractionPayload = { + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: 'approval-abc' }], + }; + expect(extractApprovalIdFromInteraction(payload)).toBe('approval-abc'); + }); + + it('returns undefined for a non-fix_it action', () => { + const payload: SlackInteractionPayload = { + type: 'block_actions', + actions: [{ action_id: 'something_else', value: 'x' }], + }; + expect(extractApprovalIdFromInteraction(payload)).toBeUndefined(); + }); + + it('returns undefined for a non-block_actions payload', () => { + expect(extractApprovalIdFromInteraction({ type: 'view_submission' })).toBeUndefined(); + }); +}); + +describe('handleSlackInteraction routes to the REAL resumeAfterApproval', () => { + let approvalStore: ApprovalStore; + let toolRegistry: ToolRegistry; + const provider = createMockProvider({ responses: ['done'] }); + + beforeEach(() => { + approvalStore = createInMemoryApprovalStore(); + toolRegistry = new ToolRegistry(); + }); + + it('calls resumeAfterApproval exactly once with the matching id, and it actually resolves the pending approval', async () => { + const resumeSpy = vi.spyOn(resumeModule, 'resumeAfterApproval'); + + const execute = vi.fn().mockResolvedValue({ fixed: true }); + toolRegistry.register('apply_fix', { + displayName: 'Apply Fix', + tool: { description: 'apply fix', parameters: {}, execute } as any, + needsApproval: true, + }); + + const pending: PendingApproval = { + id: 'approval-abc', + toolCallId: 'call-1', + toolName: 'apply_fix', + args: {}, + createdAt: new Date().toISOString(), + }; + const snapshot: ExecutionSnapshot = { + agent: { name: 'fixer', agentType: AgentType.SmartAssistant, prompt: 'fix it' }, + currentMessages: [], + pendingToolCall: pending, + steps: 1, + }; + await approvalStore.save(pending, snapshot); + + const payload: SlackInteractionPayload = { + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: 'approval-abc' }], + }; + + const result = await handleSlackInteraction(payload, { approvalStore, toolRegistry, provider }); + + expect(resumeSpy).toHaveBeenCalledTimes(1); + expect(resumeSpy.mock.calls[0][0]).toEqual({ id: 'approval-abc', approved: true }); + expect(execute).toHaveBeenCalledTimes(1); // the deferred tool actually ran + expect(result).toBeDefined(); + + resumeSpy.mockRestore(); + }); + + it('does not call resumeAfterApproval for a non-fix_it payload', async () => { + const resumeSpy = vi.spyOn(resumeModule, 'resumeAfterApproval'); + + const payload: SlackInteractionPayload = { type: 'view_submission' }; + const result = await handleSlackInteraction(payload, { approvalStore, toolRegistry, provider }); + + expect(resumeSpy).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); + + resumeSpy.mockRestore(); + }); +}); diff --git a/examples/ops-pipeline/slackInteractions.ts b/examples/ops-pipeline/slackInteractions.ts new file mode 100644 index 000000000..5665dbdba --- /dev/null +++ b/examples/ops-pipeline/slackInteractions.ts @@ -0,0 +1,151 @@ +/** + * Slack interaction-callback route (LOU-J5). + * + * Parses a Slack interaction callback (what Slack POSTs to the + * `POST /slack/interactions` endpoint when a user clicks the "Fix it" + * button built by src/tools/built-in/slack.ts's buildSlackAlertPayload()), + * extracts the approvalId from the clicked button's `value`, and calls the + * REAL LOU-C6 `resumeAfterApproval()` (src/execution/resume.ts) with that + * id - genuinely resuming the paused AgentExecutor run through the real + * ApprovalGate, not a bypass or a reimplementation of it. + */ +import * as http from 'node:http'; +import { + resumeAfterApproval, + ResumeExecuteOptions, +} from '../../src/execution/resume'; +import { ApprovalStore, ExecutionSnapshot, PendingApproval } from '../../src/execution/ApprovalGate'; +import { ExecutionResult } from '../../src/execution/AgentExecutor'; +import { LLMProvider } from '../../src/providers'; +import { ToolRegistry } from '../../src/tools'; + +/** The subset of Slack's real block_actions interaction callback payload this route reads. */ +export interface SlackInteractionPayload { + type: string; + actions?: Array<{ action_id: string; value?: string }>; +} + +/** + * Extracts the approvalId (the clicked button's `value`) from a Slack + * `block_actions` interaction payload whose action_id is 'fix_it'. Returns + * undefined if the payload isn't a recognized fix_it click. + */ +export function extractApprovalIdFromInteraction(payload: SlackInteractionPayload): string | undefined { + if (payload.type !== 'block_actions' || !payload.actions) { + return undefined; + } + const fixItAction = payload.actions.find((a) => a.action_id === 'fix_it'); + return fixItAction?.value; +} + +/** + * A simple in-memory ApprovalStore, suitable for the ops-pipeline demo and + * for tests. Mirrors the pattern src/execution/resume.test.ts uses for its + * own in-memory store. + */ +export function createInMemoryApprovalStore(): ApprovalStore { + const records = new Map(); + return { + async save(pending, snapshot) { + records.set(pending.id, { pending, snapshot }); + }, + async resolve(id) { + const record = records.get(id); + if (!record) return null; + records.delete(id); + return record; + }, + }; +} + +export interface SlackInteractionsDeps { + approvalStore: ApprovalStore; + toolRegistry: ToolRegistry; + provider: LLMProvider; + executeOptions?: ResumeExecuteOptions; +} + +/** + * Handles one parsed Slack interaction payload: extracts the approvalId and + * resumes the paused run via the REAL resumeAfterApproval(), approving the + * deferred tool call (clicking "Fix it" is an approval, not a rejection). + * Returns undefined (without calling resumeAfterApproval at all) if the + * payload isn't a recognized fix_it click. + */ +export async function handleSlackInteraction( + payload: SlackInteractionPayload, + deps: SlackInteractionsDeps +): Promise { + const approvalId = extractApprovalIdFromInteraction(payload); + if (!approvalId) { + return undefined; + } + + return resumeAfterApproval( + { id: approvalId, approved: true }, + deps.approvalStore, + deps.toolRegistry, + deps.provider, + deps.executeOptions ?? {} + ); +} + +const MAX_BODY_BYTES = 1024 * 1024; // 1MB + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + let bytes = 0; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + reject(new Error(`Request body exceeds ${MAX_BODY_BYTES} byte limit`)); + req.destroy(); + return; + } + body += chunk; + }); + req.on('end', () => resolve(body)); + req.on('error', reject); + }); +} + +/** + * Slack sends interaction callbacks as `application/x-www-form-urlencoded` + * with the JSON payload under the `payload` field - this parses either that + * form, or a raw JSON body (for convenience/tests). + */ +function parseSlackInteractionBody(raw: string): SlackInteractionPayload { + const trimmed = raw.trim(); + if (trimmed.startsWith('{')) { + return JSON.parse(trimmed); + } + const params = new URLSearchParams(raw); + const payloadField = params.get('payload'); + if (!payloadField) { + throw new Error('Slack interaction body missing "payload" field'); + } + return JSON.parse(payloadField); +} + +/** + * Builds an http.RequestListener-compatible handler for `POST + * /slack/interactions`, for use with node:http's createServer (see + * examples/ops-pipeline/index.ts). + */ +export function createSlackInteractionsRoute( + deps: SlackInteractionsDeps +): (req: http.IncomingMessage, res: http.ServerResponse) => Promise { + return async (req, res) => { + try { + const raw = await readBody(req); + const payload = parseSlackInteractionBody(raw); + const result = await handleSlackInteraction(payload, deps); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ handled: result !== undefined })); + } catch (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + }; +} diff --git a/src/tools/built-in/slack.test.ts b/src/tools/built-in/slack.test.ts new file mode 100644 index 000000000..05800fbc7 --- /dev/null +++ b/src/tools/built-in/slack.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, vi } from 'vitest'; +import { buildSlackAlertPayload, createSlackTool, postSlackAlert } from './slack'; + +function mockFetchOk() { + return vi.fn().mockResolvedValue({ + ok: true, + status: 200, + text: async () => '', + } as Response); +} + +describe('buildSlackAlertPayload', () => { + it('produces a payload with exactly one interactive button carrying the correct approvalId', () => { + const payload = buildSlackAlertPayload('#incidents', 'Deploy failed', 'approval-123'); + + const actionsBlocks = payload.blocks.filter((b) => b.type === 'actions'); + expect(actionsBlocks).toHaveLength(1); + + const buttons = actionsBlocks[0].elements ?? []; + expect(buttons).toHaveLength(1); + expect(buttons[0].action_id).toBe('fix_it'); + expect(buttons[0].value).toBe('approval-123'); + expect(buttons[0].text?.text).toBe('Fix it'); + }); + + it('includes the channel and message text', () => { + const payload = buildSlackAlertPayload('#incidents', 'Deploy failed', 'approval-123'); + expect(payload.channel).toBe('#incidents'); + expect(payload.text).toBe('Deploy failed'); + }); +}); + +describe('postSlackAlert / createSlackTool (mocked Slack API)', () => { + it('posts the built payload to the configured webhook URL', async () => { + const fetchImpl = mockFetchOk(); + + await postSlackAlert('#incidents', 'Deploy failed', 'approval-123', { + webhookUrl: 'https://hooks.slack.test/services/mock', + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe('https://hooks.slack.test/services/mock'); + const body = JSON.parse((init as RequestInit).body as string); + expect(body.blocks[1].elements[0].value).toBe('approval-123'); + }); + + it('throws a clear error when no webhook URL is configured', async () => { + const original = process.env.SLACK_WEBHOOK_URL; + delete process.env.SLACK_WEBHOOK_URL; + try { + await expect( + postSlackAlert('#incidents', 'msg', 'approval-1', { fetchImpl: mockFetchOk() as unknown as typeof fetch }) + ).rejects.toThrow(/SLACK_WEBHOOK_URL/); + } finally { + if (original !== undefined) process.env.SLACK_WEBHOOK_URL = original; + } + }); + + it('the ToolDescriptor wraps an AI SDK tool() with the expected required params', async () => { + const fetchImpl = mockFetchOk(); + const descriptor = createSlackTool({ + webhookUrl: 'https://hooks.slack.test/services/mock', + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(descriptor.tool).toBeDefined(); + const result = await descriptor.tool.execute!( + { channel: '#incidents', message: 'Deploy failed', approvalId: 'approval-123' }, + {} as any + ); + + expect(result).toEqual({ ok: true }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/tools/built-in/slack.ts b/src/tools/built-in/slack.ts new file mode 100644 index 000000000..0c99fcf6f --- /dev/null +++ b/src/tools/built-in/slack.ts @@ -0,0 +1,155 @@ +/** + * Slack alert tool (LOU-J5) + * + * Posts an alert message to a Slack channel via an Incoming Webhook, using + * Slack's Block Kit format with a single interactive "Fix it" button whose + * `value` carries an approvalId. Clicking that button is meant to resolve a + * REAL LOU-C ApprovalGate PendingApproval (see resumeAfterApproval() in + * src/execution/resume.ts, and examples/ops-pipeline/slackInteractions.ts + * for the interaction-callback route that wires the button click back to + * it) - this tool itself only sends the message; it does not bypass or + * reimplement the approval gate. + * + * Follows the same ToolDescriptor pattern established in + * src/tools/built-in/http.ts and github.ts: wraps an AI SDK `tool()` call + * (not a plain object literal). + * + * ## Env var convention (LOU-F8 style) + * + * The webhook URL is read from `SLACK_WEBHOOK_URL` by default - following + * the same "credential lives in an env var named after the provider" + * convention resolveProvider.ts's PROVIDER_ENV_TABLE uses for LLM + * providers (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`). + */ +import { tool } from 'ai'; +import { z } from 'zod'; +import { ToolDescriptor } from '../../types'; + +export const SLACK_WEBHOOK_URL_ENV_KEY = 'SLACK_WEBHOOK_URL'; + +/** + * A single Slack Block Kit block - kept intentionally minimal (only the + * shapes this tool actually emits: section and actions blocks with a + * button element) rather than modeling the whole Block Kit schema. + */ +export interface SlackBlock { + type: string; + text?: { type: string; text: string }; + elements?: Array<{ + type: string; + text?: { type: string; text: string }; + action_id?: string; + value?: string; + style?: string; + }>; +} + +export interface SlackAlertPayload { + channel: string; + text: string; + blocks: SlackBlock[]; +} + +/** + * Builds the Slack Block Kit payload for an alert with a "Fix it" button. + * Exported so callers (and tests) can inspect the exact payload shape + * without going through a real/mocked HTTP call. + */ +export function buildSlackAlertPayload( + channel: string, + message: string, + approvalId: string +): SlackAlertPayload { + return { + channel, + text: message, + blocks: [ + { + type: 'section', + text: { type: 'mrkdwn', text: message }, + }, + { + type: 'actions', + elements: [ + { + type: 'button', + text: { type: 'plain_text', text: 'Fix it' }, + action_id: 'fix_it', + value: approvalId, + style: 'primary', + }, + ], + }, + ], + }; +} + +export interface SlackToolOptions { + /** Overrides SLACK_WEBHOOK_URL for testing/injection. */ + webhookUrl?: string; + /** Overrides the fetch implementation used to POST the payload (testing). */ + fetchImpl?: typeof fetch; +} + +/** + * Posts a pre-built Slack alert payload to the configured Incoming Webhook + * URL. Exported standalone (in addition to the ToolDescriptor below) so + * examples/ops-pipeline code that isn't going through AgentExecutor's tool + * plumbing can still send an alert directly. + */ +export async function postSlackAlert( + channel: string, + message: string, + approvalId: string, + options: SlackToolOptions = {} +): Promise<{ ok: boolean }> { + const webhookUrl = options.webhookUrl ?? process.env[SLACK_WEBHOOK_URL_ENV_KEY]; + if (!webhookUrl) { + throw new Error( + `Slack tool: no webhook URL configured. Set ${SLACK_WEBHOOK_URL_ENV_KEY} or pass options.webhookUrl.` + ); + } + + const doFetch = options.fetchImpl ?? fetch; + const payload = buildSlackAlertPayload(channel, message, approvalId); + + const response = await doFetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Slack tool: webhook post failed: ${response.statusText} - ${errorText}`); + } + + return { ok: true }; +} + +/** + * Creates the Slack alert ToolDescriptor. Requires {channel, message, + * approvalId} - `approvalId` is the id of a REAL LOU-C PendingApproval + * (see ApprovalGate.ts) that the "Fix it" button, once clicked, resolves + * via resumeAfterApproval(). + */ +export function createSlackTool(options: SlackToolOptions = {}): ToolDescriptor { + return { + displayName: 'Post Slack Alert', + tool: tool({ + description: + 'Posts an alert to a Slack channel with a "Fix it" button that resumes a pending approval.', + parameters: z.object({ + channel: z.string().describe('Slack channel to post to (e.g. "#incidents")'), + message: z.string().describe('Alert message text'), + approvalId: z.string().describe('The pending approval id the "Fix it" button will resolve'), + }), + execute: async ({ channel, message, approvalId }) => { + return postSlackAlert(channel, message, approvalId, options); + }, + }), + }; +} + +/** Default Slack tool instance, reading SLACK_WEBHOOK_URL at call time. */ +export const slackTool: ToolDescriptor = createSlackTool(); From 9274057d54535ebaa5bd7e6f30abc200552fabac Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:27:35 +0300 Subject: [PATCH 06/11] LOU-J6: Fixer agent core: repo and log context Add examples/ops-pipeline/fixer.ts: FixRequest type, a fixer AgentConfig/prompt instructing diagnosis from logs and a unified diff output, runFixer() calling the real static AgentExecutor.execute() and extracting a diff block from the response (fenced ```diff, a diff-shaped plain fenced block, or a raw unified-diff marker fallback), throwing typed EmptyPatchError on an empty/unparseable patch. Also createFixerDelegateTool(), wrapping the REAL LOU-D1 createDelegateTool() so the monitor agent can delegate a FixRequest to the fixer agent via ordinary tool-calling and get back an extracted patch the same way runFixer() does directly. Co-Authored-By: Claude Sonnet 5 --- examples/ops-pipeline/fixer.test.ts | 102 ++++++++++++++++ examples/ops-pipeline/fixer.ts | 175 ++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 examples/ops-pipeline/fixer.test.ts create mode 100644 examples/ops-pipeline/fixer.ts diff --git a/examples/ops-pipeline/fixer.test.ts b/examples/ops-pipeline/fixer.test.ts new file mode 100644 index 000000000..0cbd4c362 --- /dev/null +++ b/examples/ops-pipeline/fixer.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect, vi } from 'vitest'; +import { runFixer, extractDiffBlock, EmptyPatchError, FixRequest, createFixerDelegateTool } from './fixer'; +import { LLMProvider, GenerateResult } from '../../src/providers'; + +function makeScriptedProvider(text: string): LLMProvider { + const result: GenerateResult = { + text, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 10, totalTokens: 20 }, + }; + return { + name: 'mock', + generate: vi.fn().mockResolvedValue(result), + stream: vi.fn() as any, + supportsTools: () => true, + supportsStreaming: () => true, + getModels: async () => [], + }; +} + +const request: FixRequest = { + errorSignature: 'sig-1', + logs: 'NullPointerException at OrderService.java:42', + files: ['src/OrderService.java'], +}; + +const FENCED_DIFF_RESPONSE = [ + 'Here is the fix:', + '', + '```diff', + '--- a/src/OrderService.java', + '+++ b/src/OrderService.java', + '@@ -40,3 +40,3 @@', + '- charge(null);', + '+ charge(order);', + '```', +].join('\n'); + +describe('extractDiffBlock', () => { + it('extracts a fenced ```diff block', () => { + const patch = extractDiffBlock(FENCED_DIFF_RESPONSE); + expect(patch).toContain('--- a/src/OrderService.java'); + expect(patch).toContain('+ charge(order);'); + }); + + it('extracts a diff-looking plain fenced block', () => { + const text = ['```', '--- a/foo.txt', '+++ b/foo.txt', '@@ -1 +1 @@', '-old', '+new', '```'].join( + '\n' + ); + expect(extractDiffBlock(text)).toContain('+new'); + }); + + it('falls back to scanning for unified-diff markers with no fencing', () => { + const text = ['I fixed it.', '--- a/foo.txt', '+++ b/foo.txt', '@@ -1 +1 @@', '-old', '+new'].join( + '\n' + ); + expect(extractDiffBlock(text)).toContain('+new'); + }); + + it('returns an empty string when nothing diff-shaped is present', () => { + expect(extractDiffBlock('I could not find a fix.')).toBe(''); + }); +}); + +describe('runFixer', () => { + it('with a deterministic mock provider scripted to return a fenced diff, returns a non-empty patch referencing the expected file', async () => { + const provider = makeScriptedProvider(FENCED_DIFF_RESPONSE); + + const { patch } = await runFixer(request, provider); + + expect(patch.length).toBeGreaterThan(0); + expect(patch).toContain('src/OrderService.java'); + }); + + it('throws EmptyPatchError when the fixer response has no extractable diff', async () => { + const provider = makeScriptedProvider('I am not sure how to fix this.'); + + await expect(runFixer(request, provider)).rejects.toBeInstanceOf(EmptyPatchError); + }); +}); + +describe('createFixerDelegateTool', () => { + it('delegates to the fixer agent and extracts a patch from its response', async () => { + const provider = makeScriptedProvider(FENCED_DIFF_RESPONSE); + const fixerAgent = { name: 'fixer', agentType: 0 as any, prompt: 'fix it' }; + + const delegateTool = createFixerDelegateTool({ agent: fixerAgent, provider }); + const result: any = await delegateTool.tool.execute!({ task: 'fix sig-1' }, {} as any); + + expect(result.patch).toContain('src/OrderService.java'); + }); + + it('throws EmptyPatchError when the delegated fixer agent has no extractable diff', async () => { + const provider = makeScriptedProvider('I am not sure how to fix this.'); + const fixerAgent = { name: 'fixer', agentType: 0 as any, prompt: 'fix it' }; + + const delegateTool = createFixerDelegateTool({ agent: fixerAgent, provider }); + await expect(delegateTool.tool.execute!({ task: 'fix sig-1' }, {} as any)).rejects.toBeInstanceOf( + EmptyPatchError + ); + }); +}); diff --git a/examples/ops-pipeline/fixer.ts b/examples/ops-pipeline/fixer.ts new file mode 100644 index 000000000..9920cfa21 --- /dev/null +++ b/examples/ops-pipeline/fixer.ts @@ -0,0 +1,175 @@ +/** + * Ops pipeline: fixer agent core (LOU-J6) - repo/log context in, unified + * diff out. + * + * runFixer() diagnoses a FixRequest (built from an ErrorSignal, see + * monitor.ts) via the REAL static AgentExecutor.execute() and extracts a + * unified diff from the response text. It is wired as the target of a + * LOU-D1 real createDelegateTool() call so the monitor agent can delegate a + * fix request to it by ordinary tool-calling (see createFixerDelegateTool() + * below). + * + * See guardedPr.ts (LOU-J7) for what happens to the extracted patch next + * (guardrail-gated PR creation) - kept in a sibling file since it composes + * this module's output with the LOU-E guardrails and LOU-J5 Slack tool + * rather than being fixer-core itself. + */ +import { tool } from 'ai'; +import { + AgentExecutor, + ExecuteOptions, + ExecutionResult, +} from '../../src/execution/AgentExecutor'; +import { AgentConfig, AgentType, ToolDescriptor } from '../../src/types'; +import { LLMProvider } from '../../src/providers'; +import { + createDelegateTool, + DelegateAgentOptions, +} from '../../src/execution/DelegationTool'; + +/** + * A request to diagnose-and-fix an error, built from an ErrorSignal + * (monitor.ts) or from a delegate-tool task. + */ +export interface FixRequest { + errorSignature: string; + logs: string; + files: string[]; +} + +export const FIXER_SYSTEM_PROMPT = + 'You are a fixer agent. Given error logs and the repository files involved, diagnose the root ' + + 'cause and produce a MINIMAL unified diff that fixes it. Always respond with the diff in a ' + + "fenced ```diff code block. Never invent files you were not shown."; + +/** + * Builds the fixer AgentConfig used by runFixer()/createFixerDelegateTool(). + */ +export function buildFixerAgent(name = 'fixer'): AgentConfig { + return { + name, + agentType: AgentType.SmartAssistant, + prompt: FIXER_SYSTEM_PROMPT, + }; +} + +/** + * Builds the user-message input for a FixRequest. + */ +export function buildFixerInput(request: FixRequest): string { + return [ + `Error signature: ${request.errorSignature}`, + '', + 'Logs:', + request.logs, + '', + 'Files involved:', + request.files.length > 0 ? request.files.join('\n') : '(none provided)', + ].join('\n'); +} + +/** Thrown by runFixer()/extractDiffBlock() when no non-empty diff could be extracted. */ +export class EmptyPatchError extends Error { + constructor(message = 'Fixer produced an empty or unparseable patch') { + super(message); + this.name = 'EmptyPatchError'; + } +} + +/** + * Extracts a unified diff from a fixer agent's free-text response. + * + * Strategy, in order: + * 1. A fenced ```diff ... ``` code block (or a plain ``` ... ``` block + * whose content looks like a diff) - the common case for an LLM + * response. + * 2. Falling back to scanning for unified-diff markers (`--- `, `+++ `, + * `@@`) directly in the text and taking from the first such marker to + * the end, for a response that didn't fence the diff. + * 3. Otherwise returns an empty string (callers treat this as "no patch", + * see EmptyPatchError). + */ +export function extractDiffBlock(text: string): string { + const fencedDiffMatch = text.match(/```diff\r?\n([\s\S]*?)```/); + if (fencedDiffMatch) { + return fencedDiffMatch[1].trim(); + } + + // A plain fenced block whose content itself looks like a diff. + const fencedMatches = [...text.matchAll(/```[a-zA-Z]*\r?\n([\s\S]*?)```/g)]; + for (const match of fencedMatches) { + const content = match[1]; + if (looksLikeDiff(content)) { + return content.trim(); + } + } + + const lines = text.split('\n'); + const startIndex = lines.findIndex( + (line) => line.startsWith('--- ') || line.startsWith('diff --git') || /^@@ /.test(line) + ); + if (startIndex !== -1) { + return lines.slice(startIndex).join('\n').trim(); + } + + return ''; +} + +function looksLikeDiff(content: string): boolean { + return /^(diff --git|--- |\+\+\+ |@@ )/m.test(content); +} + +/** + * Runs the fixer agent against `request` via the REAL static + * AgentExecutor.execute() and extracts a unified diff from its response. + * Throws EmptyPatchError if no non-empty patch could be extracted. + */ +export async function runFixer( + request: FixRequest, + provider: LLMProvider, + options: Partial> = {} +): Promise<{ patch: string; result: ExecutionResult }> { + const agent = buildFixerAgent(); + const result = await AgentExecutor.execute({ + ...options, + agent, + input: buildFixerInput(request), + provider, + }); + + const patch = extractDiffBlock(result.text); + if (!patch.trim()) { + throw new EmptyPatchError(); + } + + return { patch, result }; +} + +/** + * Creates a ToolDescriptor that wraps the REAL LOU-D1 createDelegateTool() + * targeting the fixer agent, post-processing its response text into an + * extracted `patch` field (via extractDiffBlock()) so a parent (monitor) + * agent's delegation to the fixer agent yields a usable patch the same way + * runFixer() does directly. Throws EmptyPatchError if the delegated fixer + * agent's response has no extractable diff. + */ +export function createFixerDelegateTool(opts: DelegateAgentOptions): ToolDescriptor { + const base = createDelegateTool(opts); + const baseTool = base.tool; + + return { + ...base, + tool: tool({ + description: baseTool.description, + parameters: baseTool.parameters, + execute: async (args: any, context: any) => { + const result = await baseTool.execute!(args, context); + const patch = extractDiffBlock((result as { text: string }).text); + if (!patch.trim()) { + throw new EmptyPatchError(); + } + return { ...(result as object), patch }; + }, + }) as typeof baseTool, + }; +} From 8e3ea1c6f0cf004b7f51435af19375719c306fe1 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:27:36 +0300 Subject: [PATCH 07/11] LOU-J7: Guardrail-gated PR creation wiring Add examples/ops-pipeline/guardedPr.ts: handleFixerPatch() runs the REAL LOU-E12 runGuardrails() (secret-scan + diff-size-cap by default) against the fixer's patch and awaits it to completion BEFORE the GitHub PR-creation call is even considered - a failing guardrail result means the GitHub tool is never called at all; instead the LOU-J5 Slack tool is called with a message naming which guardrail(s) failed, reusing the original approvalId/channel. A passing result calls the REAL LOU-E13/E14-hardened GitHub tool's PR-creation function exactly once, carrying the patch in the PR body (the only field a PR-creation-scoped token can use to hold diff content). Tests confirm: a patch matching secretScanGuardrail's real patterns, and an oversized patch, both never reach a spied GitHub PR-creation call and instead produce exactly one spied Slack call; a clean patch produces exactly one GitHub call with the patch content and zero Slack calls; and an explicit ordering test proves the guardrail check runs strictly before the GitHub call (sequential, not parallel/best-effort). Co-Authored-By: Claude Sonnet 5 --- examples/ops-pipeline/guardedPr.test.ts | 119 ++++++++++++++++++++++++ examples/ops-pipeline/guardedPr.ts | 103 ++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 examples/ops-pipeline/guardedPr.test.ts create mode 100644 examples/ops-pipeline/guardedPr.ts diff --git a/examples/ops-pipeline/guardedPr.test.ts b/examples/ops-pipeline/guardedPr.test.ts new file mode 100644 index 000000000..97443ac55 --- /dev/null +++ b/examples/ops-pipeline/guardedPr.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect, vi } from 'vitest'; +import { handleFixerPatch } from './guardedPr'; +import { ToolDescriptor } from '../../src/types'; + +describe('handleFixerPatch (LOU-J7)', () => { + const CLEAN_PATCH = '--- a/foo.txt\n+++ b/foo.txt\n@@ -1 +1 @@\n-old\n+new\n'; + const SECRET_PATCH = `--- a/config.txt\n+++ b/config.txt\n@@ -1 +1 @@\n-old\n+AKIA${'A'.repeat(16)}\n`; + + function makeSpiedTool(name: string): ToolDescriptor & { execute: ReturnType } { + const execute = vi.fn().mockResolvedValue({ ok: true, tool: name }); + return { + displayName: name, + tool: { description: name, parameters: {}, execute } as any, + execute, + } as any; + } + + it('a clean patch results in exactly one GitHub PR-creation call with the patch content, and never calls Slack', async () => { + const githubCreatePrTool = makeSpiedTool('github_create_pull_request'); + const slackTool = makeSpiedTool('slack'); + + const result = await handleFixerPatch(CLEAN_PATCH, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-1', + head: 'fix/auto', + }); + + expect(result.pass).toBe(true); + expect(githubCreatePrTool.execute).toHaveBeenCalledTimes(1); + expect(slackTool.execute).not.toHaveBeenCalled(); + + const prArgs = githubCreatePrTool.execute.mock.calls[0][0]; + expect(prArgs.body).toContain('+new'); + }); + + it('a patch containing a fake API-key-shaped string never reaches the GitHub tool, and Slack is notified instead', async () => { + const githubCreatePrTool = makeSpiedTool('github_create_pull_request'); + const slackTool = makeSpiedTool('slack'); + + const result = await handleFixerPatch(SECRET_PATCH, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-2', + head: 'fix/auto', + }); + + expect(result.pass).toBe(false); + expect(githubCreatePrTool.execute).not.toHaveBeenCalled(); + expect(slackTool.execute).toHaveBeenCalledTimes(1); + + const slackArgs = slackTool.execute.mock.calls[0][0]; + expect(slackArgs.approvalId).toBe('approval-2'); + expect(slackArgs.message).toContain('secret-scan'); + }); + + it('a patch exceeding the diff-size cap never reaches the GitHub tool either', async () => { + const githubCreatePrTool = makeSpiedTool('github_create_pull_request'); + const slackTool = makeSpiedTool('slack'); + + const bigPatch = Array.from({ length: 600 }, (_, i) => `+line ${i}`).join('\n'); + + const result = await handleFixerPatch(bigPatch, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-4', + head: 'fix/auto', + }); + + expect(result.pass).toBe(false); + expect(githubCreatePrTool.execute).not.toHaveBeenCalled(); + expect(slackTool.execute).toHaveBeenCalledTimes(1); + }); + + it('the guardrail check completes before the GitHub call is even considered (sequential, not parallel)', async () => { + const order: string[] = []; + const githubCreatePrTool: any = { + displayName: 'github', + tool: { + description: 'github', + parameters: {}, + execute: vi.fn(async () => { + order.push('github'); + return { ok: true }; + }), + }, + }; + const slackTool: any = { + displayName: 'slack', + tool: { description: 'slack', parameters: {}, execute: vi.fn() }, + }; + + const { createDiffSizeGuardrail, secretScanGuardrail } = await import('../../src/execution/guardrails'); + const trackedGuardrails = [ + secretScanGuardrail, + { + name: 'diff-size-cap', + check: async (action: { diff: string }) => { + order.push('guardrail'); + return createDiffSizeGuardrail(500).check(action); + }, + }, + ]; + + await handleFixerPatch(CLEAN_PATCH, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-3', + head: 'fix/auto', + guardrails: trackedGuardrails as any, + }); + + expect(order).toEqual(['guardrail', 'github']); + }); +}); diff --git a/examples/ops-pipeline/guardedPr.ts b/examples/ops-pipeline/guardedPr.ts new file mode 100644 index 000000000..d96af438f --- /dev/null +++ b/examples/ops-pipeline/guardedPr.ts @@ -0,0 +1,103 @@ +/** + * Ops pipeline: guardrail-gated PR creation wiring (LOU-J7). + * + * After fixer.ts's runFixer() returns {patch}, handleFixerPatch() gates + * that patch through the REAL LOU-E12 runGuardrails() (secret-scan + + * diff-size-cap by default) BEFORE ever considering the GitHub + * PR-creation call. If the guardrail result's `pass` is false, the GitHub + * tool is NEVER called - instead the LOU-J5 Slack tool is called with a + * message naming which guardrail(s) failed and why, reusing the original + * approvalId/channel. If `pass` is true, the REAL LOU-E13/E14-hardened + * GitHub tool's PR-creation function is called exactly once with the + * patch as diff content. + * + * This sequence is synchronous/sequential - the guardrail check is + * `await`ed to completion before the GitHub call is even considered (no + * Promise.all/race with it) - so the guardrail gate is genuinely blocking, + * not parallel or best-effort. Per this epic's own Implementation + * Considerations, this gate is not configurable off: callers may swap + * which guardrails run, but handleFixerPatch() always runs the guardrail + * check before any GitHub call, with no way to skip it. + */ +import { + Guardrail, + ProposedAction, + createDiffSizeGuardrail, + runGuardrails, + secretScanGuardrail, +} from '../../src/execution/guardrails'; +import { ToolDescriptor } from '../../src/types'; + +export interface FixerPatchContext { + /** The REAL LOU-E13/E14-hardened GitHub tool's PR-creation ToolDescriptor. */ + githubCreatePrTool: ToolDescriptor; + /** The LOU-J5 Slack tool used to report a guardrail failure. */ + slackTool: ToolDescriptor; + /** Slack channel to notify on guardrail failure. */ + channel: string; + /** The original approvalId/thread reference, reused for the Slack failure message. */ + approvalId: string; + head: string; + base?: string; + title?: string; + /** + * Guardrails to run before ever considering the GitHub call. Defaults to + * the REAL secretScanGuardrail plus a 500-line createDiffSizeGuardrail - + * both real, non-configurable-off gates per this epic's own + * Implementation Considerations. + */ + guardrails?: Guardrail[]; +} + +export interface HandleFixerPatchResult { + pass: boolean; + failures?: { name: string; reason?: string }[]; + prResult?: unknown; +} + +/** + * Runs the REAL runGuardrails() against `patch` and, ONLY if it passes, + * calls the REAL GitHub PR-creation tool exactly once with the patch as + * diff content (a PR-creation-scoped GitHub token has no endpoint to + * attach a raw diff directly to a PR - see github.ts's scope + * documentation - so the patch is carried in the PR body, the only field + * available to a PR-creation/reading-scoped token that can hold it). + */ +export async function handleFixerPatch( + patch: string, + context: FixerPatchContext +): Promise { + const guardrails = context.guardrails ?? [secretScanGuardrail, createDiffSizeGuardrail(500)]; + const action: ProposedAction = { diff: patch }; + + const guardrailResult = await runGuardrails(action, guardrails); + + if (!guardrailResult.pass) { + const reasons = guardrailResult.failures + .map((f) => `${f.name}${f.reason ? `: ${f.reason}` : ''}`) + .join('; '); + + await context.slackTool.tool.execute!( + { + channel: context.channel, + message: `Guardrail check failed, PR not created (approval ${context.approvalId}): ${reasons}`, + approvalId: context.approvalId, + }, + {} as any + ); + + return { pass: false, failures: guardrailResult.failures }; + } + + const prResult = await context.githubCreatePrTool.tool.execute!( + { + title: context.title ?? `Automated fix (approval ${context.approvalId})`, + body: `Automated fix generated by the ops-pipeline fixer agent.\n\n\`\`\`diff\n${patch}\n\`\`\``, + head: context.head, + base: context.base, + }, + {} as any + ); + + return { pass: true, prResult }; +} From 968ce86201a5c5972f7a3f28b93499147e616052 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:32:37 +0300 Subject: [PATCH 08/11] LOU-J8: End-to-end demo packaging Add examples/ops-pipeline/index.ts wiring, in one process entry point: the LOU-J4 monitor webhook listener, a POST /slack/interactions route (LOU-J5), the LOU-J6 fixer (delegated to via a REAL createDelegateTool() flagged needsApproval:true), and the LOU-J7 guardrail-gated PR wiring. The delegate tool's approval requirement is the load-bearing safety property here: the fixer agent's AgentExecutor.execute() call only happens from inside that tool's execute(), which only runs after resumeAfterApproval() (LOU-C6) resolves a real PendingApproval - there is no path that runs the fixer without that approval round-trip. Add examples/ops-pipeline/mocks/ (mock Grafana/Datadog sender, mock Slack API, mock GitHub API - same call shapes as the real tools) and a deterministic zero-network demoProvider.ts, so the whole example runs with zero external network access by default. Add pipeline:demo / pipeline:demo:trigger npm scripts and a README with numbered setup steps. Verified genuinely end-to-end from a fresh `npm run pipeline:demo` process plus `npm run pipeline:demo:trigger` in a second terminal, and via a committed integration test (index.test.ts) that drives the full dedup -> approval-gate -> Slack "Fix it" -> guardrail-gated GitHub PR sequence and asserts each step's call counts. Co-Authored-By: Claude Sonnet 5 --- examples/ops-pipeline/.env.example | 17 ++ examples/ops-pipeline/README.md | 97 +++++++ examples/ops-pipeline/demoProvider.ts | 106 +++++++ examples/ops-pipeline/index.test.ts | 70 +++++ examples/ops-pipeline/index.ts | 262 ++++++++++++++++++ examples/ops-pipeline/mocks/mockGithubTool.ts | 48 ++++ .../ops-pipeline/mocks/mockGrafanaSender.ts | 30 ++ examples/ops-pipeline/mocks/mockSlackTool.ts | 49 ++++ .../ops-pipeline/mocks/sendSyntheticError.ts | 24 ++ package.json | 2 + 10 files changed, 705 insertions(+) create mode 100644 examples/ops-pipeline/.env.example create mode 100644 examples/ops-pipeline/README.md create mode 100644 examples/ops-pipeline/demoProvider.ts create mode 100644 examples/ops-pipeline/index.test.ts create mode 100644 examples/ops-pipeline/index.ts create mode 100644 examples/ops-pipeline/mocks/mockGithubTool.ts create mode 100644 examples/ops-pipeline/mocks/mockGrafanaSender.ts create mode 100644 examples/ops-pipeline/mocks/mockSlackTool.ts create mode 100644 examples/ops-pipeline/mocks/sendSyntheticError.ts diff --git a/examples/ops-pipeline/.env.example b/examples/ops-pipeline/.env.example new file mode 100644 index 000000000..f65ac4262 --- /dev/null +++ b/examples/ops-pipeline/.env.example @@ -0,0 +1,17 @@ +# All of these are OPTIONAL - the default `npm run pipeline:demo` runs +# entirely against mocks/ and needs none of them set. + +# Real Slack Incoming Webhook URL (src/tools/built-in/slack.ts). +SLACK_WEBHOOK_URL= + +# Real LLM provider credentials, only used if you wire a real LLMProvider +# into startOpsPipeline({ provider }) instead of the default demoProvider. +OPENAI_API_KEY= +ANTHROPIC_API_KEY= + +# Real GitHub token, owner/repo, only used if you wire a real GitHubTools +# instance into startOpsPipeline({ githubCreatePrTool }) instead of the +# default mock GitHub tool. +GITHUB_TOKEN= +GITHUB_OWNER= +GITHUB_REPO= diff --git a/examples/ops-pipeline/README.md b/examples/ops-pipeline/README.md new file mode 100644 index 000000000..e40c09774 --- /dev/null +++ b/examples/ops-pipeline/README.md @@ -0,0 +1,97 @@ +# ops-pipeline (LOU-J8) + +An end-to-end "flagship pipeline" demo composing every LOU-J primitive: + +1. **Monitor** (LOU-J4) - a `POST /webhook` listener that receives a Grafana/ + Datadog-shaped `ErrorSignal`, dedupes it by signature, and asks the + monitor agent to triage it. +2. **Approval gate** (LOU-C, via the real `ApprovalGate`/`resumeAfterApproval`) + - the monitor's delegation to the fixer agent is flagged + `needsApproval: true`, so it always pauses for a human before the fixer + agent ever runs. +3. **Slack "Fix it" button** (LOU-J5) - the pause posts a Slack alert with a + "Fix it" button; clicking it (`POST /slack/interactions`) resumes the + paused run through the REAL `resumeAfterApproval()`. +4. **Fixer** (LOU-J6) - once approved, the fixer agent diagnoses the error + and produces a unified diff. +5. **Guardrail-gated PR** (LOU-J7) - the diff is checked against the REAL + `secretScanGuardrail` + a diff-size cap BEFORE a GitHub PR is even + considered; a failure notifies Slack instead of ever calling GitHub. + +By default this runs entirely against the `mocks/` implementations (mock +Slack API, mock GitHub API, and a deterministic scripted LLM provider), so +it needs **zero external network access** and zero API keys. + +## Setup + +1. Install dependencies (from the repo root): + + ``` + npm install + ``` + +2. Copy the example env file (only needed if you want to point this at + REAL Slack/GitHub/LLM providers instead of the mocks - the demo works + with none of this set): + + ``` + cp examples/ops-pipeline/.env.example .env + ``` + +3. Start the demo pipeline: + + ``` + npm run pipeline:demo + ``` + + This starts two local listeners (both bound to `127.0.0.1`): + - `http://127.0.0.1:8787/webhook` - the monitor's error-signal intake + - `http://127.0.0.1:8788/slack/interactions` - the Slack button callback + +4. In a second terminal, POST a synthetic error at the running pipeline: + + ``` + npm run pipeline:demo:trigger + ``` + + Watch the first terminal's output - this triggers the monitor agent, + which pauses for approval and posts a (mock) Slack alert with a "Fix + it" button. + +5. Simulate clicking "Fix it" (in the real integration this is a real + Slack button click; here it's a plain POST simulating Slack's + interaction callback - substitute the `approvalId` your mock Slack + tool logged): + + ``` + curl -X POST http://127.0.0.1:8788/slack/interactions \ + -H 'Content-Type: application/json' \ + -d '{"type":"block_actions","actions":[{"action_id":"fix_it","value":""}]}' + ``` + + This resumes the paused run, runs the fixer agent, and - if the + guardrail check passes - creates a (mock) GitHub pull request. + +## Files + +- `monitor.ts` - LOU-J4: ErrorSignal type, dedup, `POST /webhook`. +- `fixer.ts` - LOU-J6: fixer agent, diff extraction, delegate-tool wiring. +- `guardedPr.ts` - LOU-J7: guardrail-gated PR creation. +- `demoProvider.ts` - a deterministic, zero-network scripted LLM provider + used by the demo in place of a real model. +- `mocks/` - mock Grafana sender, mock Slack API, mock GitHub API (same + call shapes as the real tools). +- `index.ts` - wires everything into one process (`npm run pipeline:demo`). + +## Real credentials (optional) + +Set these to run against real services instead of the mocks - none are +required for the default demo: + +- `SLACK_WEBHOOK_URL` - real Slack Incoming Webhook (see + `src/tools/built-in/slack.ts`) +- `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` - real LLM provider (pass a real + `LLMProvider` to `startOpsPipeline({ provider })` instead of the default + `demoProvider`) +- A real GitHub token (`src/tools/built-in/github.ts`'s `createGitHubTools`) + passed as `githubCreatePrTool` to `startOpsPipeline()` diff --git a/examples/ops-pipeline/demoProvider.ts b/examples/ops-pipeline/demoProvider.ts new file mode 100644 index 000000000..c0fffaeed --- /dev/null +++ b/examples/ops-pipeline/demoProvider.ts @@ -0,0 +1,106 @@ +/** + * Deterministic, zero-network scripted LLMProvider for the ops-pipeline + * demo (LOU-J8). Not a "real" provider - it inspects the conversation + * shape to decide what a real model would plausibly do at each step: + * + * - Monitor agent turn (tools present, no tool result yet in the + * conversation): calls the `delegate_to_fixer` tool with the + * triggering user message as the delegated task. + * - Monitor agent's follow-up turn (tools present, a tool result already + * in the conversation): finishes with a short confirmation, no further + * tool calls. + * - Fixer agent turn (no tools - the fixer agent is never given a tool + * registry): returns a fenced ```diff block diagnosing the demo's + * synthetic NullPointerException. + */ +import { GenerateOptions, GenerateResult, LLMProvider, StreamResult } from '../../src/providers'; + +export const DEMO_FIXER_DIFF_RESPONSE = [ + 'Root cause: `order` is not null-checked before `charge()` is called.', + '', + '```diff', + '--- a/src/OrderService.java', + '+++ b/src/OrderService.java', + '@@ -39,7 +39,9 @@', + ' public void checkout(Order order) {', + '- charge(order.getPayment());', + '+ if (order.getPayment() == null) {', + '+ throw new IllegalArgumentException("payment is required");', + '+ }', + '+ charge(order.getPayment());', + ' }', + '```', +].join('\n'); + +function usageFor(text: string): GenerateResult['usage'] { + const tokens = Math.max(1, Math.ceil(text.length / 4)); + return { promptTokens: tokens, completionTokens: tokens, totalTokens: tokens * 2 }; +} + +export function createDemoProvider(): LLMProvider { + const generate = async (options: GenerateOptions): Promise => { + const hasTools = !!options.tools && options.tools.length > 0; + const alreadyDelegated = options.messages.some((m) => m.role === 'tool'); + + if (hasTools && !alreadyDelegated) { + const lastUser = [...options.messages].reverse().find((m) => m.role === 'user'); + return { + text: '', + finishReason: 'tool_calls', + usage: usageFor(lastUser?.content ?? ''), + toolCalls: [ + { + id: 'call_delegate_1', + type: 'function', + function: { + name: 'delegate_to_fixer', + arguments: JSON.stringify({ task: lastUser?.content ?? '' }), + }, + }, + ], + }; + } + + if (hasTools && alreadyDelegated) { + const text = 'Delegated to the fixer agent for review.'; + return { text, finishReason: 'stop', usage: usageFor(text) }; + } + + // Fixer agent turn: no tools registered for it. + return { + text: DEMO_FIXER_DIFF_RESPONSE, + finishReason: 'stop', + usage: usageFor(DEMO_FIXER_DIFF_RESPONSE), + }; + }; + + return { + name: 'ops-pipeline-demo', + generate, + stream: async (options: GenerateOptions): Promise => { + const result = await generate(options); + async function* full() { + yield { type: 'text-delta' as const, textDelta: result.text }; + yield { + type: 'finish' as const, + finishReason: result.finishReason, + usage: result.usage, + }; + } + async function* textOnly() { + yield result.text; + } + return { + fullStream: full(), + textStream: textOnly(), + text: Promise.resolve(result.text), + usage: Promise.resolve(result.usage), + finishReason: Promise.resolve(result.finishReason), + toolCalls: Promise.resolve(result.toolCalls ?? []), + }; + }, + supportsTools: () => true, + supportsStreaming: () => true, + getModels: async () => ['ops-pipeline-demo'], + }; +} diff --git a/examples/ops-pipeline/index.test.ts b/examples/ops-pipeline/index.test.ts new file mode 100644 index 000000000..44dff59c3 --- /dev/null +++ b/examples/ops-pipeline/index.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { startOpsPipeline, OpsPipelineHandle } from './index'; +import { createDemoProvider } from './demoProvider'; +import { createMockGithubTool } from './mocks/mockGithubTool'; +import { createMockSlackTool } from './mocks/mockSlackTool'; +import { createInMemoryApprovalStore } from './slackInteractions'; +import { exampleErrorSignal, sendSyntheticError } from './mocks/mockGrafanaSender'; + +describe('ops-pipeline end-to-end (LOU-J8, against mocks - zero external network)', () => { + let handle: OpsPipelineHandle | undefined; + + afterEach(async () => { + if (handle) { + await handle.close(); + handle = undefined; + } + }); + + it('runs the full pipeline: dedup -> approval gate -> Slack "Fix it" -> guardrail-gated GitHub PR', async () => { + const provider = createDemoProvider(); + const github = createMockGithubTool(); + const slack = createMockSlackTool(); + const approvalStore = createInMemoryApprovalStore(); + + handle = await startOpsPipeline({ + provider, + githubCreatePrTool: github.tool, + slackTool: slack.tool, + approvalStore, + monitorPort: 0, + slackPort: 0, + }); + + const signal = exampleErrorSignal(); + const webhookUrl = `http://127.0.0.1:${handle.monitor.port}/webhook`; + + // New signature is accepted; a duplicate is deduped. + const res1 = await sendSyntheticError(webhookUrl, signal); + expect((await res1.json()).deduped).toBe(false); + const res2 = await sendSyntheticError(webhookUrl, signal); + expect((await res2.json()).deduped).toBe(true); + + // Let the async onResult -> Slack post settle. + await new Promise((resolve) => setTimeout(resolve, 50)); + + // The fixer must NOT have run / no PR yet - blocked behind the approval gate. + expect(slack.posts).toHaveLength(1); + expect(github.createdPullRequests).toHaveLength(0); + + const approvalId = slack.posts[0].approvalId; + expect(approvalId).toBeTruthy(); + + // Simulate the Slack "Fix it" button click. + const interactionsUrl = `http://127.0.0.1:${handle.slack.port}/slack/interactions`; + const interactionRes = await fetch(interactionsUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: approvalId }], + }), + }); + expect((await interactionRes.json()).handled).toBe(true); + + // The approval unblocked the fixer, which produced a patch that passed + // guardrails and resulted in exactly one GitHub PR. + expect(github.createdPullRequests).toHaveLength(1); + expect(github.createdPullRequests[0].body).toContain('OrderService.java'); + }); +}); diff --git a/examples/ops-pipeline/index.ts b/examples/ops-pipeline/index.ts new file mode 100644 index 000000000..66550d380 --- /dev/null +++ b/examples/ops-pipeline/index.ts @@ -0,0 +1,262 @@ +/** + * Ops pipeline (LOU-J8) - end-to-end demo entry point. + * + * Wires, in order: + * 1. LOU-J4's monitor webhook listener (POST /webhook) + * 2. LOU-J5's Slack tool + interaction route (POST /slack/interactions) + * 3. LOU-J6's fixer, delegated to from the monitor agent + * 4. LOU-J7's guardrail-gated PR creation + * into one process. + * + * Safety property this wiring enforces (per the epic's own Implementation + * Considerations): the fixer agent never runs without first passing + * through the REAL LOU-C approval gate. The `delegate_to_fixer` tool + * registered on the monitor agent is marked `needsApproval: true`, so + * AgentExecutor.execute() pauses and saves a PendingApproval (via the REAL + * ApprovalGate/ApprovalStore) the moment the monitor agent tries to call + * it - the fixer agent's AgentExecutor.execute() call is only ever reached + * from inside that tool's execute(), which only runs after a human clicks + * "Fix it" in Slack and resumeAfterApproval() (LOU-C6, called for real by + * slackInteractions.ts) resolves the approval and runs the deferred tool + * call. There is no code path here that runs the fixer without that + * approval round-trip. + * + * Defaults to the mocks/ implementations (zero external network access); + * pass real tool descriptors/provider to wire it to a real Slack webhook, + * GitHub token and LLM provider instead. + */ +import * as http from 'node:http'; +import { AgentType, ToolDescriptor } from '../../src/types'; +import { ToolRegistry } from '../../src/tools'; +import { LLMProvider } from '../../src/providers/llm'; +import { startMonitorServer, MonitorServerHandle } from './monitor'; +import { createFixerDelegateTool, buildFixerAgent } from './fixer'; +import { handleFixerPatch } from './guardedPr'; +import { + createInMemoryApprovalStore, + extractApprovalIdFromInteraction, + SlackInteractionPayload, + handleSlackInteraction, +} from './slackInteractions'; +import { ApprovalStore } from '../../src/execution/ApprovalGate'; +import { ExecutionResult } from '../../src/execution/AgentExecutor'; +import { createDemoProvider } from './demoProvider'; +import { createMockGithubTool } from './mocks/mockGithubTool'; +import { createMockSlackTool } from './mocks/mockSlackTool'; + +const DELEGATE_TOOL_NAME = 'delegate_to_fixer'; +const DEFAULT_CHANNEL = '#incidents'; + +/** + * Pulls the fixer's extracted `patch` back out of a resumed ExecutionResult + * by finding the delegate tool's own result message (AgentExecutor appends + * it as a `role: 'tool'` message whose content is + * `JSON.stringify(toolResult.result)`, and createFixerDelegateTool's + * result shape includes a `patch` field - see fixer.ts). + */ +function extractDelegatedPatch(result: ExecutionResult, toolName: string): string | undefined { + for (let i = result.messages.length - 1; i >= 0; i--) { + const message = result.messages[i]; + if (message.role === 'tool' && message.toolName === toolName) { + try { + const parsed = JSON.parse(message.content); + if (typeof parsed?.patch === 'string') { + return parsed.patch; + } + } catch { + // Not JSON / no patch field - keep scanning older messages. + } + } + } + return undefined; +} + +export interface OpsPipelineDeps { + provider?: LLMProvider; + githubCreatePrTool?: ToolDescriptor; + slackTool?: ToolDescriptor; + approvalStore?: ApprovalStore; + monitorHost?: string; + monitorPort?: number; + slackHost?: string; + slackPort?: number; + channel?: string; +} + +export interface OpsPipelineHandle { + monitor: MonitorServerHandle; + slack: { server: http.Server; port: number; close: () => Promise }; + close: () => Promise; +} + +const MAX_BODY_BYTES = 1024 * 1024; + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + let bytes = 0; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + reject(new Error(`Request body exceeds ${MAX_BODY_BYTES} byte limit`)); + req.destroy(); + return; + } + body += chunk; + }); + req.on('end', () => resolve(body)); + req.on('error', reject); + }); +} + +function parseSlackInteractionBody(raw: string): SlackInteractionPayload { + const trimmed = raw.trim(); + if (trimmed.startsWith('{')) { + return JSON.parse(trimmed); + } + const params = new URLSearchParams(raw); + const payloadField = params.get('payload'); + if (!payloadField) { + throw new Error('Slack interaction body missing "payload" field'); + } + return JSON.parse(payloadField); +} + +/** + * Starts the full ops pipeline: the monitor webhook server, and a second + * small server exposing POST /slack/interactions (kept as a separate + * listener from the monitor server rather than merged into one + * http.Server, since startMonitorServer() already owns its own server + * lifecycle from LOU-J4 - both are still started from this one process + * entry point). + */ +export async function startOpsPipeline(deps: OpsPipelineDeps = {}): Promise { + const provider = deps.provider ?? createDemoProvider(); + const githubCreatePrTool = deps.githubCreatePrTool ?? createMockGithubTool().tool; + const slackTool = deps.slackTool ?? createMockSlackTool().tool; + const approvalStore = deps.approvalStore ?? createInMemoryApprovalStore(); + const channel = deps.channel ?? DEFAULT_CHANNEL; + + const fixerAgent = buildFixerAgent(); + const delegateTool = createFixerDelegateTool({ agent: fixerAgent, provider }); + // The fixer agent must never run without first passing through the + // approval gate (this epic's own non-negotiable safety property) - + // enforced here by flagging the delegate tool itself as needing + // approval, so AgentExecutor.execute() pauses BEFORE ever calling it. + const gatedDelegateTool: ToolDescriptor = { ...delegateTool, needsApproval: true }; + + const toolRegistry = new ToolRegistry(); + toolRegistry.register(DELEGATE_TOOL_NAME, gatedDelegateTool); + + const monitorAgent = { + name: 'ops-monitor', + agentType: AgentType.SmartAssistant, + prompt: + 'You are an ops monitor. When an alert fires, delegate it to the fixer agent via the ' + + `${DELEGATE_TOOL_NAME} tool.`, + tools: { [DELEGATE_TOOL_NAME]: { tool: DELEGATE_TOOL_NAME } }, + }; + + const monitor = await startMonitorServer({ + executeOptions: { + agent: monitorAgent, + provider, + toolRegistry, + approvalStore, + }, + host: deps.monitorHost, + port: deps.monitorPort, + onResult: async (signal, result) => { + if (result.finishReason === 'awaiting-approval' && result.approvalId) { + await slackTool.tool.execute!( + { + channel, + message: `New error detected (${signal.signature}): ${signal.message}. Approve the fix?`, + approvalId: result.approvalId, + }, + {} as any + ); + } + }, + }); + + const slackServer = http.createServer((req, res) => { + void (async () => { + if (req.method === 'POST' && req.url === '/slack/interactions') { + try { + const raw = await readBody(req); + const payload = parseSlackInteractionBody(raw); + const approvalId = extractApprovalIdFromInteraction(payload); + + const result = await handleSlackInteraction(payload, { + approvalStore, + toolRegistry, + provider, + }); + + if (result && approvalId) { + const patch = extractDelegatedPatch(result, DELEGATE_TOOL_NAME); + if (patch) { + await handleFixerPatch(patch, { + githubCreatePrTool, + slackTool, + channel, + approvalId, + head: `fix/auto-${approvalId}`, + }); + } + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ handled: result !== undefined })); + } catch (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + return; + } + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + })(); + }); + + const slackHost = deps.slackHost ?? '127.0.0.1'; + const slackPort = deps.slackPort ?? 0; + await new Promise((resolve, reject) => { + slackServer.once('error', reject); + slackServer.listen(slackPort, slackHost, () => resolve()); + }); + const slackAddress = slackServer.address(); + const actualSlackPort = typeof slackAddress === 'object' && slackAddress ? slackAddress.port : slackPort; + + return { + monitor, + slack: { + server: slackServer, + port: actualSlackPort, + close: () => new Promise((resolve, reject) => slackServer.close((e) => (e ? reject(e) : resolve()))), + }, + close: async () => { + await monitor.close(); + await new Promise((resolve, reject) => slackServer.close((e) => (e ? reject(e) : resolve()))); + }, + }; +} + +if (require.main === module) { + const monitorPort = Number(process.env.OPS_PIPELINE_MONITOR_PORT) || 8787; + const slackPort = Number(process.env.OPS_PIPELINE_SLACK_PORT) || 8788; + + startOpsPipeline({ monitorPort, slackPort }) + .then((handle) => { + console.log(`ops-pipeline monitor webhook listening on http://127.0.0.1:${handle.monitor.port}/webhook`); + console.log( + `ops-pipeline slack interactions listening on http://127.0.0.1:${handle.slack.port}/slack/interactions` + ); + console.log('Running against mock Slack/GitHub APIs and a deterministic demo provider by default.'); + }) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/examples/ops-pipeline/mocks/mockGithubTool.ts b/examples/ops-pipeline/mocks/mockGithubTool.ts new file mode 100644 index 000000000..90ddfc876 --- /dev/null +++ b/examples/ops-pipeline/mocks/mockGithubTool.ts @@ -0,0 +1,48 @@ +/** + * Mock GitHub API (LOU-J8) - same call shape as the REAL github tool's + * `github_create_pull_request` (src/tools/built-in/github.ts): takes + * {title, body, head, base}, returns JSON matching {number, url, state}. + * Never makes a real network call, so the ops-pipeline demo runs with zero + * external network access. + */ +import { ToolDescriptor } from '../../../src/types'; + +export interface MockCreatedPullRequest { + title: string; + body: string; + head: string; + base?: string; +} + +export interface MockGithubTool { + tool: ToolDescriptor; + createdPullRequests: MockCreatedPullRequest[]; +} + +export function createMockGithubTool(): MockGithubTool { + const createdPullRequests: MockCreatedPullRequest[] = []; + let nextNumber = 1; + + const tool: ToolDescriptor = { + displayName: 'Create GitHub Pull Request (mock)', + tool: { + description: 'Mock: creates a pull request', + parameters: {} as any, + execute: async (args: MockCreatedPullRequest) => { + createdPullRequests.push(args); + const number = nextNumber++; + return JSON.stringify( + { + number, + url: `https://github.com/mock-org/mock-repo/pull/${number}`, + state: 'open', + }, + null, + 2 + ); + }, + } as any, + }; + + return { tool, createdPullRequests }; +} diff --git a/examples/ops-pipeline/mocks/mockGrafanaSender.ts b/examples/ops-pipeline/mocks/mockGrafanaSender.ts new file mode 100644 index 000000000..fc759a49e --- /dev/null +++ b/examples/ops-pipeline/mocks/mockGrafanaSender.ts @@ -0,0 +1,30 @@ +/** + * Mock Grafana/Datadog webhook sender (LOU-J8) - same call shape as a real + * monitoring source: a plain HTTP POST of an ErrorSignal JSON body to the + * pipeline's /webhook endpoint. Used by the demo's "post a synthetic + * error" script; posts only to the demo's own localhost server, so this + * never reaches the real network either. + */ +import { ErrorSignal } from '../monitor'; + +export async function sendSyntheticError(webhookUrl: string, signal: ErrorSignal): Promise { + return fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(signal), + }); +} + +/** A representative synthetic error signal for the demo. */ +export function exampleErrorSignal(): ErrorSignal { + return { + signature: 'demo-npe-order-service', + service: 'grafana', + message: 'NullPointerException in OrderService.charge', + logs: [ + 'java.lang.NullPointerException', + '\tat OrderService.charge(OrderService.java:42)', + '\tat OrderController.checkout(OrderController.java:18)', + ].join('\n'), + }; +} diff --git a/examples/ops-pipeline/mocks/mockSlackTool.ts b/examples/ops-pipeline/mocks/mockSlackTool.ts new file mode 100644 index 000000000..117a13d16 --- /dev/null +++ b/examples/ops-pipeline/mocks/mockSlackTool.ts @@ -0,0 +1,49 @@ +/** + * Mock Slack API (LOU-J8) - same call shape as the REAL Slack tool + * (src/tools/built-in/slack.ts's createSlackTool()): takes + * {channel, message, approvalId}, builds the exact same Block Kit payload + * via the real buildSlackAlertPayload(), and records it instead of + * POSTing to a real webhook - so the ops-pipeline demo runs with zero + * external network access. + */ +import { ToolDescriptor } from '../../../src/types'; +import { buildSlackAlertPayload, SlackAlertPayload } from '../../../src/tools/built-in/slack'; + +export interface MockSlackPost { + channel: string; + message: string; + approvalId: string; + payload: SlackAlertPayload; +} + +export interface MockSlackTool { + tool: ToolDescriptor; + posts: MockSlackPost[]; +} + +export function createMockSlackTool(): MockSlackTool { + const posts: MockSlackPost[] = []; + + const tool: ToolDescriptor = { + displayName: 'Post Slack Alert (mock)', + tool: { + description: 'Mock: posts a Slack alert with a Fix it button', + parameters: {} as any, + execute: async ({ + channel, + message, + approvalId, + }: { + channel: string; + message: string; + approvalId: string; + }) => { + const payload = buildSlackAlertPayload(channel, message, approvalId); + posts.push({ channel, message, approvalId, payload }); + return { ok: true }; + }, + } as any, + }; + + return { tool, posts }; +} diff --git a/examples/ops-pipeline/mocks/sendSyntheticError.ts b/examples/ops-pipeline/mocks/sendSyntheticError.ts new file mode 100644 index 000000000..6f2bb2de0 --- /dev/null +++ b/examples/ops-pipeline/mocks/sendSyntheticError.ts @@ -0,0 +1,24 @@ +/** + * Standalone script: POSTs a synthetic ErrorSignal at the running + * ops-pipeline demo's /webhook endpoint (LOU-J8 README step). + * + * Usage: tsx examples/ops-pipeline/mocks/sendSyntheticError.ts [port] + * (defaults to port 8787, matching index.ts's default monitor port) + */ +import { sendSyntheticError, exampleErrorSignal } from './mockGrafanaSender'; + +async function main() { + const port = Number(process.argv[2]) || 8787; + const url = `http://127.0.0.1:${port}/webhook`; + const signal = exampleErrorSignal(); + + console.log(`POSTing synthetic error signature "${signal.signature}" to ${url} ...`); + const response = await sendSyntheticError(url, signal); + const body = await response.json(); + console.log(`Response ${response.status}:`, body); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/package.json b/package.json index d3a9e5943..83b23b615 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,8 @@ "example:workflow-router": "tsx examples/workflow-router/index.ts", "example:doc-qa": "tsx examples/doc-qa/index.ts", "example:slack-notifier": "tsx examples/slack-notifier/index.ts", + "pipeline:demo": "tsx examples/ops-pipeline/index.ts", + "pipeline:demo:trigger": "tsx examples/ops-pipeline/mocks/sendSyntheticError.ts", "prepublishOnly": "npm run build", "docs:build": "typedoc", "docs:verify-snippets": "tsx scripts/verify-docs-snippets.ts" From 72c227c2955fcd6376ab163ecc9088f79b97aaaa Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:34:20 +0300 Subject: [PATCH 09/11] LOU-J9: Pipeline eval Add examples/ops-pipeline/pipeline.eval.ts with two complementary checks: 1. A real defineEval() (LOU-G1) call scored with the real toolCallOrder() scorer (LOU-G4) against the monitor agent's own single AgentExecutor.execute() step, asserting it calls the real 'delegate_to_fixer' tool name used by index.ts's wiring. 2. A direct, spy-instrumented assertion of the full cross-stage call order (monitor's Slack alert -> both real guardrails -> GitHub PR creation) - chosen because most of this pipeline's calls are plain ToolDescriptor invocations outside AgentExecutor's own tool-calling loop, so they never populate one single ExecutionResult.toolCalls array for toolCallOrder() to score directly; a deep-equals over the recorded order is the more honest representation of what genuinely happened. Also threads an optional `guardrails` override through startOpsPipeline() so this eval (and any other test) can observe guardrail execution order without altering index.ts's own default guardrails. Verified the eval is a real assertion, not a no-op: ran it passing, then temporarily reordered EXPECTED_ORDER and confirmed it correctly failed with a clear diff, then restored the correct order. Co-Authored-By: Claude Sonnet 5 --- examples/ops-pipeline/index.ts | 10 ++ examples/ops-pipeline/pipeline.eval.ts | 172 +++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 examples/ops-pipeline/pipeline.eval.ts diff --git a/examples/ops-pipeline/index.ts b/examples/ops-pipeline/index.ts index 66550d380..029fe6841 100644 --- a/examples/ops-pipeline/index.ts +++ b/examples/ops-pipeline/index.ts @@ -40,6 +40,7 @@ import { } from './slackInteractions'; import { ApprovalStore } from '../../src/execution/ApprovalGate'; import { ExecutionResult } from '../../src/execution/AgentExecutor'; +import { Guardrail } from '../../src/execution/guardrails'; import { createDemoProvider } from './demoProvider'; import { createMockGithubTool } from './mocks/mockGithubTool'; import { createMockSlackTool } from './mocks/mockSlackTool'; @@ -81,6 +82,14 @@ export interface OpsPipelineDeps { slackHost?: string; slackPort?: number; channel?: string; + /** + * Guardrails to run before considering the GitHub PR call. Defaults to + * handleFixerPatch()'s own defaults (real secretScanGuardrail + a + * diff-size cap) when omitted. Exposed here so callers (e.g. LOU-J9's + * pipeline.eval.ts) can wrap the default guardrails to observe/record + * execution order without changing which guardrails actually run. + */ + guardrails?: Guardrail[]; } export interface OpsPipelineHandle { @@ -203,6 +212,7 @@ export async function startOpsPipeline(deps: OpsPipelineDeps = {}): Promise Slack alert -> (approval) -> fixer delegate + * -> guardrails -> GitHub PR creation, most of which do NOT flow + * through one single AgentExecutor.execute() call's `toolCalls` field + * (the Slack/GitHub calls are plain ToolDescriptor invocations made + * directly by index.ts/guardedPr.ts, not LLM-issued tool calls). A + * defineEval()+toolCallOrder() call cannot express that cross-stage + * sequence, so - per this ticket's own guidance - it's instead recorded + * directly: every real stage (the monitor's delegate tool call, the + * Slack alert, both real guardrails, and GitHub PR creation) is + * instrumented with the same spy pattern fixer.test.ts/guardedPr.test.ts/ + * index.test.ts already use, and the recorded order is asserted with a + * plain deep-equals - the most honest representation of what actually + * ran, in what order, without inventing a fake single-result shape. + */ +import { describe, it, expect } from 'vitest'; +import { defineEval } from '../../src/evals/defineEval'; +import { toolCallOrder } from '../../src/evals/scorers'; +import { AgentType } from '../../src/types'; +import { ToolRegistry } from '../../src/tools'; +import { secretScanGuardrail, createDiffSizeGuardrail } from '../../src/execution/guardrails'; +import { startOpsPipeline } from './index'; +import { createDemoProvider } from './demoProvider'; +import { createFixerDelegateTool, buildFixerAgent } from './fixer'; +import { buildMonitorPrompt } from './monitor'; +import { createMockGithubTool } from './mocks/mockGithubTool'; +import { createMockSlackTool } from './mocks/mockSlackTool'; +import { createInMemoryApprovalStore } from './slackInteractions'; +import { exampleErrorSignal, sendSyntheticError } from './mocks/mockGrafanaSender'; + +const DELEGATE_TOOL_NAME = 'delegate_to_fixer'; +const FIXTURE_SIGNAL = exampleErrorSignal(); + +// --------------------------------------------------------------------------- +// 1. REAL defineEval() + REAL toolCallOrder() scorer (LOU-G1/G4) +// --------------------------------------------------------------------------- + +const evalProvider = createDemoProvider(); +const evalFixerAgent = buildFixerAgent(); +const evalDelegateTool = createFixerDelegateTool({ agent: evalFixerAgent, provider: evalProvider }); +// Note: unlike index.ts's real wiring, this eval registry does NOT flag the +// delegate tool needsApproval:true - defineEval()'s EvalConfig (LOU-G1) has +// no approvalStore field to satisfy a paused run, and the approval-gate +// property itself is already covered directly by index.test.ts. This eval +// is purely about the monitor agent's own tool-call CHOICE/ORDER. +const evalToolRegistry = new ToolRegistry(); +evalToolRegistry.register(DELEGATE_TOOL_NAME, evalDelegateTool); + +const evalMonitorAgent = { + name: 'ops-monitor', + agentType: AgentType.SmartAssistant, + prompt: `You are an ops monitor. When an alert fires, delegate it to the fixer agent via the ${DELEGATE_TOOL_NAME} tool.`, + tools: { [DELEGATE_TOOL_NAME]: { tool: DELEGATE_TOOL_NAME } }, +}; + +defineEval({ + name: 'ops-pipeline: monitor triages a new error by calling delegate_to_fixer exactly once', + agent: evalMonitorAgent, + input: buildMonitorPrompt(FIXTURE_SIGNAL), + provider: evalProvider, + toolRegistry: evalToolRegistry, + score: toolCallOrder([{ tool: DELEGATE_TOOL_NAME }]), + threshold: 1, +}); + +// --------------------------------------------------------------------------- +// 2. Full cross-stage call-order recording (direct assertion) +// --------------------------------------------------------------------------- + +describe('ops-pipeline: full cross-stage call order (LOU-J9)', () => { + it('runs monitor -> slack alert -> fixer delegate -> guardrails -> github PR, in that order', async () => { + const order: string[] = []; + + const provider = createDemoProvider(); + const approvalStore = createInMemoryApprovalStore(); + + const github = createMockGithubTool(); + const slack = createMockSlackTool(); + + // Wrap the mock tools' execute() to record call order without changing + // their real behavior/return shape. + const githubCreatePrTool = { + ...github.tool, + tool: { + ...github.tool.tool, + execute: async (...args: Parameters>) => { + order.push('github:create_pull_request'); + return github.tool.tool.execute!(...args); + }, + }, + }; + const slackTool = { + ...slack.tool, + tool: { + ...slack.tool.tool, + execute: async (...args: Parameters>) => { + order.push('slack:alert'); + return slack.tool.tool.execute!(...args); + }, + }, + }; + + const trackedGuardrails = [ + { + name: 'secret-scan', + check: async (action: { diff: string }) => { + order.push('guardrail:secret-scan'); + return secretScanGuardrail.check(action); + }, + }, + { + name: 'diff-size-cap', + check: async (action: { diff: string }) => { + order.push('guardrail:diff-size-cap'); + return createDiffSizeGuardrail(500).check(action); + }, + }, + ]; + + const handle = await startOpsPipeline({ + provider, + githubCreatePrTool: githubCreatePrTool as any, + slackTool: slackTool as any, + approvalStore, + guardrails: trackedGuardrails as any, + monitorPort: 0, + slackPort: 0, + }); + + try { + const webhookUrl = `http://127.0.0.1:${handle.monitor.port}/webhook`; + await sendSyntheticError(webhookUrl, FIXTURE_SIGNAL); + await new Promise((resolve) => setTimeout(resolve, 50)); // let the monitor's onResult -> Slack alert settle + + const approvalId = slack.posts[0]?.approvalId; + expect(approvalId).toBeTruthy(); + + const interactionsUrl = `http://127.0.0.1:${handle.slack.port}/slack/interactions`; + await fetch(interactionsUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: approvalId }], + }), + }); + + const EXPECTED_ORDER = [ + 'slack:alert', + 'guardrail:secret-scan', + 'guardrail:diff-size-cap', + 'github:create_pull_request', + ]; + + expect(order).toEqual(EXPECTED_ORDER); + } finally { + await handle.close(); + } + }); +}); From e13e19953f80cb4a60f69e857c2903883b287d98 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:36:11 +0300 Subject: [PATCH 10/11] LOU-J8 fix: add ops-pipeline to examples/README.md gallery index examples/README.test.ts asserts the gallery index has exactly one heading per example directory - LOU-J8's new examples/ops-pipeline/ directory needed a matching entry. Co-Authored-By: Claude Sonnet 5 --- examples/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/README.md b/examples/README.md index e90c73924..44e0241a4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,6 +9,12 @@ to try. A question-answering agent scoped to a single fixed document (LOU-H10). +## [ops-pipeline](./ops-pipeline) + +A flagship end-to-end pipeline: a Grafana/Datadog monitor delegates a fix (behind a real +human approval gate, with a Slack "Fix it" button) to a fixer agent, whose patch is +guardrail-gated before a GitHub PR is opened (LOU-J4-J9). + ## [research-assistant](./research-assistant) A research agent with the built-in `http` tool wired up (LOU-H10). From 6acdfa527d666e7e52d22ca6a3c44eec6ba64cec Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Mon, 28 Sep 2026 23:44:46 +0300 Subject: [PATCH 11/11] LOU-J fix: address two non-blocking QA notes on the ops-pipeline demo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fixer.ts: document runFixer()'s safety contract explicitly (it produces a patch candidate only — no approval gate, no guardrail check — so a future caller wiring it into a new entry point doesn't accidentally bypass the approval/guardrail gates the shipped pipeline relies on). - mockSlackTool.ts: log the approvalId and alert message to the console when the demo posts an alert, since the README's "click Fix it" step otherwise has no way to surface the approvalId in a live terminal run (the mock has no real Slack UI to click a button in). QA found no exploitable bypass in either area — these are documentation/ observability hardening, not fixes to an actual gap in the safety gates. Co-Authored-By: Claude Sonnet 5 --- examples/ops-pipeline/fixer.ts | 8 ++++++++ examples/ops-pipeline/mocks/mockSlackTool.ts | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/examples/ops-pipeline/fixer.ts b/examples/ops-pipeline/fixer.ts index 9920cfa21..72cc251b6 100644 --- a/examples/ops-pipeline/fixer.ts +++ b/examples/ops-pipeline/fixer.ts @@ -123,6 +123,14 @@ function looksLikeDiff(content: string): boolean { * Runs the fixer agent against `request` via the REAL static * AgentExecutor.execute() and extracts a unified diff from its response. * Throws EmptyPatchError if no non-empty patch could be extracted. + * + * SAFETY: this function only produces a patch candidate. It does NOT run + * guardrails and does NOT gate on human approval. In the shipped pipeline + * (index.ts) it is only ever reached from inside the delegate tool that + * AgentExecutor pauses on `needsApproval: true` before invoking. Do not + * call runFixer() directly from a new entry point without first routing + * through that same approval gate and through handleFixerPatch()'s + * guardrail check (guardedPr.ts) before any GitHub write action. */ export async function runFixer( request: FixRequest, diff --git a/examples/ops-pipeline/mocks/mockSlackTool.ts b/examples/ops-pipeline/mocks/mockSlackTool.ts index 117a13d16..c7075c74b 100644 --- a/examples/ops-pipeline/mocks/mockSlackTool.ts +++ b/examples/ops-pipeline/mocks/mockSlackTool.ts @@ -40,6 +40,11 @@ export function createMockSlackTool(): MockSlackTool { }) => { const payload = buildSlackAlertPayload(channel, message, approvalId); posts.push({ channel, message, approvalId, payload }); + // Logged so the README's manual "click Fix it" step is actually + // followable when running the live demo from the terminal - the + // mock has no real Slack UI to click a button in. + console.log(`[mock-slack] alert posted to ${channel} — approvalId: ${approvalId}`); + console.log(`[mock-slack] ${message}`); return { ok: true }; }, } as any,