diff --git a/examples/README.md b/examples/README.md index e90c73924..44e0241a4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,6 +9,12 @@ to try. A question-answering agent scoped to a single fixed document (LOU-H10). +## [ops-pipeline](./ops-pipeline) + +A flagship end-to-end pipeline: a Grafana/Datadog monitor delegates a fix (behind a real +human approval gate, with a Slack "Fix it" button) to a fixer agent, whose patch is +guardrail-gated before a GitHub PR is opened (LOU-J4-J9). + ## [research-assistant](./research-assistant) A research agent with the built-in `http` tool wired up (LOU-H10). diff --git a/examples/ops-pipeline/.env.example b/examples/ops-pipeline/.env.example new file mode 100644 index 000000000..f65ac4262 --- /dev/null +++ b/examples/ops-pipeline/.env.example @@ -0,0 +1,17 @@ +# All of these are OPTIONAL - the default `npm run pipeline:demo` runs +# entirely against mocks/ and needs none of them set. + +# Real Slack Incoming Webhook URL (src/tools/built-in/slack.ts). +SLACK_WEBHOOK_URL= + +# Real LLM provider credentials, only used if you wire a real LLMProvider +# into startOpsPipeline({ provider }) instead of the default demoProvider. +OPENAI_API_KEY= +ANTHROPIC_API_KEY= + +# Real GitHub token, owner/repo, only used if you wire a real GitHubTools +# instance into startOpsPipeline({ githubCreatePrTool }) instead of the +# default mock GitHub tool. +GITHUB_TOKEN= +GITHUB_OWNER= +GITHUB_REPO= diff --git a/examples/ops-pipeline/README.md b/examples/ops-pipeline/README.md new file mode 100644 index 000000000..e40c09774 --- /dev/null +++ b/examples/ops-pipeline/README.md @@ -0,0 +1,97 @@ +# ops-pipeline (LOU-J8) + +An end-to-end "flagship pipeline" demo composing every LOU-J primitive: + +1. **Monitor** (LOU-J4) - a `POST /webhook` listener that receives a Grafana/ + Datadog-shaped `ErrorSignal`, dedupes it by signature, and asks the + monitor agent to triage it. +2. **Approval gate** (LOU-C, via the real `ApprovalGate`/`resumeAfterApproval`) + - the monitor's delegation to the fixer agent is flagged + `needsApproval: true`, so it always pauses for a human before the fixer + agent ever runs. +3. **Slack "Fix it" button** (LOU-J5) - the pause posts a Slack alert with a + "Fix it" button; clicking it (`POST /slack/interactions`) resumes the + paused run through the REAL `resumeAfterApproval()`. +4. **Fixer** (LOU-J6) - once approved, the fixer agent diagnoses the error + and produces a unified diff. +5. **Guardrail-gated PR** (LOU-J7) - the diff is checked against the REAL + `secretScanGuardrail` + a diff-size cap BEFORE a GitHub PR is even + considered; a failure notifies Slack instead of ever calling GitHub. + +By default this runs entirely against the `mocks/` implementations (mock +Slack API, mock GitHub API, and a deterministic scripted LLM provider), so +it needs **zero external network access** and zero API keys. + +## Setup + +1. Install dependencies (from the repo root): + + ``` + npm install + ``` + +2. Copy the example env file (only needed if you want to point this at + REAL Slack/GitHub/LLM providers instead of the mocks - the demo works + with none of this set): + + ``` + cp examples/ops-pipeline/.env.example .env + ``` + +3. Start the demo pipeline: + + ``` + npm run pipeline:demo + ``` + + This starts two local listeners (both bound to `127.0.0.1`): + - `http://127.0.0.1:8787/webhook` - the monitor's error-signal intake + - `http://127.0.0.1:8788/slack/interactions` - the Slack button callback + +4. In a second terminal, POST a synthetic error at the running pipeline: + + ``` + npm run pipeline:demo:trigger + ``` + + Watch the first terminal's output - this triggers the monitor agent, + which pauses for approval and posts a (mock) Slack alert with a "Fix + it" button. + +5. Simulate clicking "Fix it" (in the real integration this is a real + Slack button click; here it's a plain POST simulating Slack's + interaction callback - substitute the `approvalId` your mock Slack + tool logged): + + ``` + curl -X POST http://127.0.0.1:8788/slack/interactions \ + -H 'Content-Type: application/json' \ + -d '{"type":"block_actions","actions":[{"action_id":"fix_it","value":""}]}' + ``` + + This resumes the paused run, runs the fixer agent, and - if the + guardrail check passes - creates a (mock) GitHub pull request. + +## Files + +- `monitor.ts` - LOU-J4: ErrorSignal type, dedup, `POST /webhook`. +- `fixer.ts` - LOU-J6: fixer agent, diff extraction, delegate-tool wiring. +- `guardedPr.ts` - LOU-J7: guardrail-gated PR creation. +- `demoProvider.ts` - a deterministic, zero-network scripted LLM provider + used by the demo in place of a real model. +- `mocks/` - mock Grafana sender, mock Slack API, mock GitHub API (same + call shapes as the real tools). +- `index.ts` - wires everything into one process (`npm run pipeline:demo`). + +## Real credentials (optional) + +Set these to run against real services instead of the mocks - none are +required for the default demo: + +- `SLACK_WEBHOOK_URL` - real Slack Incoming Webhook (see + `src/tools/built-in/slack.ts`) +- `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` - real LLM provider (pass a real + `LLMProvider` to `startOpsPipeline({ provider })` instead of the default + `demoProvider`) +- A real GitHub token (`src/tools/built-in/github.ts`'s `createGitHubTools`) + passed as `githubCreatePrTool` to `startOpsPipeline()` diff --git a/examples/ops-pipeline/demoProvider.ts b/examples/ops-pipeline/demoProvider.ts new file mode 100644 index 000000000..c0fffaeed --- /dev/null +++ b/examples/ops-pipeline/demoProvider.ts @@ -0,0 +1,106 @@ +/** + * Deterministic, zero-network scripted LLMProvider for the ops-pipeline + * demo (LOU-J8). Not a "real" provider - it inspects the conversation + * shape to decide what a real model would plausibly do at each step: + * + * - Monitor agent turn (tools present, no tool result yet in the + * conversation): calls the `delegate_to_fixer` tool with the + * triggering user message as the delegated task. + * - Monitor agent's follow-up turn (tools present, a tool result already + * in the conversation): finishes with a short confirmation, no further + * tool calls. + * - Fixer agent turn (no tools - the fixer agent is never given a tool + * registry): returns a fenced ```diff block diagnosing the demo's + * synthetic NullPointerException. + */ +import { GenerateOptions, GenerateResult, LLMProvider, StreamResult } from '../../src/providers'; + +export const DEMO_FIXER_DIFF_RESPONSE = [ + 'Root cause: `order` is not null-checked before `charge()` is called.', + '', + '```diff', + '--- a/src/OrderService.java', + '+++ b/src/OrderService.java', + '@@ -39,7 +39,9 @@', + ' public void checkout(Order order) {', + '- charge(order.getPayment());', + '+ if (order.getPayment() == null) {', + '+ throw new IllegalArgumentException("payment is required");', + '+ }', + '+ charge(order.getPayment());', + ' }', + '```', +].join('\n'); + +function usageFor(text: string): GenerateResult['usage'] { + const tokens = Math.max(1, Math.ceil(text.length / 4)); + return { promptTokens: tokens, completionTokens: tokens, totalTokens: tokens * 2 }; +} + +export function createDemoProvider(): LLMProvider { + const generate = async (options: GenerateOptions): Promise => { + const hasTools = !!options.tools && options.tools.length > 0; + const alreadyDelegated = options.messages.some((m) => m.role === 'tool'); + + if (hasTools && !alreadyDelegated) { + const lastUser = [...options.messages].reverse().find((m) => m.role === 'user'); + return { + text: '', + finishReason: 'tool_calls', + usage: usageFor(lastUser?.content ?? ''), + toolCalls: [ + { + id: 'call_delegate_1', + type: 'function', + function: { + name: 'delegate_to_fixer', + arguments: JSON.stringify({ task: lastUser?.content ?? '' }), + }, + }, + ], + }; + } + + if (hasTools && alreadyDelegated) { + const text = 'Delegated to the fixer agent for review.'; + return { text, finishReason: 'stop', usage: usageFor(text) }; + } + + // Fixer agent turn: no tools registered for it. + return { + text: DEMO_FIXER_DIFF_RESPONSE, + finishReason: 'stop', + usage: usageFor(DEMO_FIXER_DIFF_RESPONSE), + }; + }; + + return { + name: 'ops-pipeline-demo', + generate, + stream: async (options: GenerateOptions): Promise => { + const result = await generate(options); + async function* full() { + yield { type: 'text-delta' as const, textDelta: result.text }; + yield { + type: 'finish' as const, + finishReason: result.finishReason, + usage: result.usage, + }; + } + async function* textOnly() { + yield result.text; + } + return { + fullStream: full(), + textStream: textOnly(), + text: Promise.resolve(result.text), + usage: Promise.resolve(result.usage), + finishReason: Promise.resolve(result.finishReason), + toolCalls: Promise.resolve(result.toolCalls ?? []), + }; + }, + supportsTools: () => true, + supportsStreaming: () => true, + getModels: async () => ['ops-pipeline-demo'], + }; +} diff --git a/examples/ops-pipeline/fixer.test.ts b/examples/ops-pipeline/fixer.test.ts new file mode 100644 index 000000000..0cbd4c362 --- /dev/null +++ b/examples/ops-pipeline/fixer.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect, vi } from 'vitest'; +import { runFixer, extractDiffBlock, EmptyPatchError, FixRequest, createFixerDelegateTool } from './fixer'; +import { LLMProvider, GenerateResult } from '../../src/providers'; + +function makeScriptedProvider(text: string): LLMProvider { + const result: GenerateResult = { + text, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 10, totalTokens: 20 }, + }; + return { + name: 'mock', + generate: vi.fn().mockResolvedValue(result), + stream: vi.fn() as any, + supportsTools: () => true, + supportsStreaming: () => true, + getModels: async () => [], + }; +} + +const request: FixRequest = { + errorSignature: 'sig-1', + logs: 'NullPointerException at OrderService.java:42', + files: ['src/OrderService.java'], +}; + +const FENCED_DIFF_RESPONSE = [ + 'Here is the fix:', + '', + '```diff', + '--- a/src/OrderService.java', + '+++ b/src/OrderService.java', + '@@ -40,3 +40,3 @@', + '- charge(null);', + '+ charge(order);', + '```', +].join('\n'); + +describe('extractDiffBlock', () => { + it('extracts a fenced ```diff block', () => { + const patch = extractDiffBlock(FENCED_DIFF_RESPONSE); + expect(patch).toContain('--- a/src/OrderService.java'); + expect(patch).toContain('+ charge(order);'); + }); + + it('extracts a diff-looking plain fenced block', () => { + const text = ['```', '--- a/foo.txt', '+++ b/foo.txt', '@@ -1 +1 @@', '-old', '+new', '```'].join( + '\n' + ); + expect(extractDiffBlock(text)).toContain('+new'); + }); + + it('falls back to scanning for unified-diff markers with no fencing', () => { + const text = ['I fixed it.', '--- a/foo.txt', '+++ b/foo.txt', '@@ -1 +1 @@', '-old', '+new'].join( + '\n' + ); + expect(extractDiffBlock(text)).toContain('+new'); + }); + + it('returns an empty string when nothing diff-shaped is present', () => { + expect(extractDiffBlock('I could not find a fix.')).toBe(''); + }); +}); + +describe('runFixer', () => { + it('with a deterministic mock provider scripted to return a fenced diff, returns a non-empty patch referencing the expected file', async () => { + const provider = makeScriptedProvider(FENCED_DIFF_RESPONSE); + + const { patch } = await runFixer(request, provider); + + expect(patch.length).toBeGreaterThan(0); + expect(patch).toContain('src/OrderService.java'); + }); + + it('throws EmptyPatchError when the fixer response has no extractable diff', async () => { + const provider = makeScriptedProvider('I am not sure how to fix this.'); + + await expect(runFixer(request, provider)).rejects.toBeInstanceOf(EmptyPatchError); + }); +}); + +describe('createFixerDelegateTool', () => { + it('delegates to the fixer agent and extracts a patch from its response', async () => { + const provider = makeScriptedProvider(FENCED_DIFF_RESPONSE); + const fixerAgent = { name: 'fixer', agentType: 0 as any, prompt: 'fix it' }; + + const delegateTool = createFixerDelegateTool({ agent: fixerAgent, provider }); + const result: any = await delegateTool.tool.execute!({ task: 'fix sig-1' }, {} as any); + + expect(result.patch).toContain('src/OrderService.java'); + }); + + it('throws EmptyPatchError when the delegated fixer agent has no extractable diff', async () => { + const provider = makeScriptedProvider('I am not sure how to fix this.'); + const fixerAgent = { name: 'fixer', agentType: 0 as any, prompt: 'fix it' }; + + const delegateTool = createFixerDelegateTool({ agent: fixerAgent, provider }); + await expect(delegateTool.tool.execute!({ task: 'fix sig-1' }, {} as any)).rejects.toBeInstanceOf( + EmptyPatchError + ); + }); +}); diff --git a/examples/ops-pipeline/fixer.ts b/examples/ops-pipeline/fixer.ts new file mode 100644 index 000000000..72cc251b6 --- /dev/null +++ b/examples/ops-pipeline/fixer.ts @@ -0,0 +1,183 @@ +/** + * Ops pipeline: fixer agent core (LOU-J6) - repo/log context in, unified + * diff out. + * + * runFixer() diagnoses a FixRequest (built from an ErrorSignal, see + * monitor.ts) via the REAL static AgentExecutor.execute() and extracts a + * unified diff from the response text. It is wired as the target of a + * LOU-D1 real createDelegateTool() call so the monitor agent can delegate a + * fix request to it by ordinary tool-calling (see createFixerDelegateTool() + * below). + * + * See guardedPr.ts (LOU-J7) for what happens to the extracted patch next + * (guardrail-gated PR creation) - kept in a sibling file since it composes + * this module's output with the LOU-E guardrails and LOU-J5 Slack tool + * rather than being fixer-core itself. + */ +import { tool } from 'ai'; +import { + AgentExecutor, + ExecuteOptions, + ExecutionResult, +} from '../../src/execution/AgentExecutor'; +import { AgentConfig, AgentType, ToolDescriptor } from '../../src/types'; +import { LLMProvider } from '../../src/providers'; +import { + createDelegateTool, + DelegateAgentOptions, +} from '../../src/execution/DelegationTool'; + +/** + * A request to diagnose-and-fix an error, built from an ErrorSignal + * (monitor.ts) or from a delegate-tool task. + */ +export interface FixRequest { + errorSignature: string; + logs: string; + files: string[]; +} + +export const FIXER_SYSTEM_PROMPT = + 'You are a fixer agent. Given error logs and the repository files involved, diagnose the root ' + + 'cause and produce a MINIMAL unified diff that fixes it. Always respond with the diff in a ' + + "fenced ```diff code block. Never invent files you were not shown."; + +/** + * Builds the fixer AgentConfig used by runFixer()/createFixerDelegateTool(). + */ +export function buildFixerAgent(name = 'fixer'): AgentConfig { + return { + name, + agentType: AgentType.SmartAssistant, + prompt: FIXER_SYSTEM_PROMPT, + }; +} + +/** + * Builds the user-message input for a FixRequest. + */ +export function buildFixerInput(request: FixRequest): string { + return [ + `Error signature: ${request.errorSignature}`, + '', + 'Logs:', + request.logs, + '', + 'Files involved:', + request.files.length > 0 ? request.files.join('\n') : '(none provided)', + ].join('\n'); +} + +/** Thrown by runFixer()/extractDiffBlock() when no non-empty diff could be extracted. */ +export class EmptyPatchError extends Error { + constructor(message = 'Fixer produced an empty or unparseable patch') { + super(message); + this.name = 'EmptyPatchError'; + } +} + +/** + * Extracts a unified diff from a fixer agent's free-text response. + * + * Strategy, in order: + * 1. A fenced ```diff ... ``` code block (or a plain ``` ... ``` block + * whose content looks like a diff) - the common case for an LLM + * response. + * 2. Falling back to scanning for unified-diff markers (`--- `, `+++ `, + * `@@`) directly in the text and taking from the first such marker to + * the end, for a response that didn't fence the diff. + * 3. Otherwise returns an empty string (callers treat this as "no patch", + * see EmptyPatchError). + */ +export function extractDiffBlock(text: string): string { + const fencedDiffMatch = text.match(/```diff\r?\n([\s\S]*?)```/); + if (fencedDiffMatch) { + return fencedDiffMatch[1].trim(); + } + + // A plain fenced block whose content itself looks like a diff. + const fencedMatches = [...text.matchAll(/```[a-zA-Z]*\r?\n([\s\S]*?)```/g)]; + for (const match of fencedMatches) { + const content = match[1]; + if (looksLikeDiff(content)) { + return content.trim(); + } + } + + const lines = text.split('\n'); + const startIndex = lines.findIndex( + (line) => line.startsWith('--- ') || line.startsWith('diff --git') || /^@@ /.test(line) + ); + if (startIndex !== -1) { + return lines.slice(startIndex).join('\n').trim(); + } + + return ''; +} + +function looksLikeDiff(content: string): boolean { + return /^(diff --git|--- |\+\+\+ |@@ )/m.test(content); +} + +/** + * Runs the fixer agent against `request` via the REAL static + * AgentExecutor.execute() and extracts a unified diff from its response. + * Throws EmptyPatchError if no non-empty patch could be extracted. + * + * SAFETY: this function only produces a patch candidate. It does NOT run + * guardrails and does NOT gate on human approval. In the shipped pipeline + * (index.ts) it is only ever reached from inside the delegate tool that + * AgentExecutor pauses on `needsApproval: true` before invoking. Do not + * call runFixer() directly from a new entry point without first routing + * through that same approval gate and through handleFixerPatch()'s + * guardrail check (guardedPr.ts) before any GitHub write action. + */ +export async function runFixer( + request: FixRequest, + provider: LLMProvider, + options: Partial> = {} +): Promise<{ patch: string; result: ExecutionResult }> { + const agent = buildFixerAgent(); + const result = await AgentExecutor.execute({ + ...options, + agent, + input: buildFixerInput(request), + provider, + }); + + const patch = extractDiffBlock(result.text); + if (!patch.trim()) { + throw new EmptyPatchError(); + } + + return { patch, result }; +} + +/** + * Creates a ToolDescriptor that wraps the REAL LOU-D1 createDelegateTool() + * targeting the fixer agent, post-processing its response text into an + * extracted `patch` field (via extractDiffBlock()) so a parent (monitor) + * agent's delegation to the fixer agent yields a usable patch the same way + * runFixer() does directly. Throws EmptyPatchError if the delegated fixer + * agent's response has no extractable diff. + */ +export function createFixerDelegateTool(opts: DelegateAgentOptions): ToolDescriptor { + const base = createDelegateTool(opts); + const baseTool = base.tool; + + return { + ...base, + tool: tool({ + description: baseTool.description, + parameters: baseTool.parameters, + execute: async (args: any, context: any) => { + const result = await baseTool.execute!(args, context); + const patch = extractDiffBlock((result as { text: string }).text); + if (!patch.trim()) { + throw new EmptyPatchError(); + } + return { ...(result as object), patch }; + }, + }) as typeof baseTool, + }; +} diff --git a/examples/ops-pipeline/guardedPr.test.ts b/examples/ops-pipeline/guardedPr.test.ts new file mode 100644 index 000000000..97443ac55 --- /dev/null +++ b/examples/ops-pipeline/guardedPr.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect, vi } from 'vitest'; +import { handleFixerPatch } from './guardedPr'; +import { ToolDescriptor } from '../../src/types'; + +describe('handleFixerPatch (LOU-J7)', () => { + const CLEAN_PATCH = '--- a/foo.txt\n+++ b/foo.txt\n@@ -1 +1 @@\n-old\n+new\n'; + const SECRET_PATCH = `--- a/config.txt\n+++ b/config.txt\n@@ -1 +1 @@\n-old\n+AKIA${'A'.repeat(16)}\n`; + + function makeSpiedTool(name: string): ToolDescriptor & { execute: ReturnType } { + const execute = vi.fn().mockResolvedValue({ ok: true, tool: name }); + return { + displayName: name, + tool: { description: name, parameters: {}, execute } as any, + execute, + } as any; + } + + it('a clean patch results in exactly one GitHub PR-creation call with the patch content, and never calls Slack', async () => { + const githubCreatePrTool = makeSpiedTool('github_create_pull_request'); + const slackTool = makeSpiedTool('slack'); + + const result = await handleFixerPatch(CLEAN_PATCH, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-1', + head: 'fix/auto', + }); + + expect(result.pass).toBe(true); + expect(githubCreatePrTool.execute).toHaveBeenCalledTimes(1); + expect(slackTool.execute).not.toHaveBeenCalled(); + + const prArgs = githubCreatePrTool.execute.mock.calls[0][0]; + expect(prArgs.body).toContain('+new'); + }); + + it('a patch containing a fake API-key-shaped string never reaches the GitHub tool, and Slack is notified instead', async () => { + const githubCreatePrTool = makeSpiedTool('github_create_pull_request'); + const slackTool = makeSpiedTool('slack'); + + const result = await handleFixerPatch(SECRET_PATCH, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-2', + head: 'fix/auto', + }); + + expect(result.pass).toBe(false); + expect(githubCreatePrTool.execute).not.toHaveBeenCalled(); + expect(slackTool.execute).toHaveBeenCalledTimes(1); + + const slackArgs = slackTool.execute.mock.calls[0][0]; + expect(slackArgs.approvalId).toBe('approval-2'); + expect(slackArgs.message).toContain('secret-scan'); + }); + + it('a patch exceeding the diff-size cap never reaches the GitHub tool either', async () => { + const githubCreatePrTool = makeSpiedTool('github_create_pull_request'); + const slackTool = makeSpiedTool('slack'); + + const bigPatch = Array.from({ length: 600 }, (_, i) => `+line ${i}`).join('\n'); + + const result = await handleFixerPatch(bigPatch, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-4', + head: 'fix/auto', + }); + + expect(result.pass).toBe(false); + expect(githubCreatePrTool.execute).not.toHaveBeenCalled(); + expect(slackTool.execute).toHaveBeenCalledTimes(1); + }); + + it('the guardrail check completes before the GitHub call is even considered (sequential, not parallel)', async () => { + const order: string[] = []; + const githubCreatePrTool: any = { + displayName: 'github', + tool: { + description: 'github', + parameters: {}, + execute: vi.fn(async () => { + order.push('github'); + return { ok: true }; + }), + }, + }; + const slackTool: any = { + displayName: 'slack', + tool: { description: 'slack', parameters: {}, execute: vi.fn() }, + }; + + const { createDiffSizeGuardrail, secretScanGuardrail } = await import('../../src/execution/guardrails'); + const trackedGuardrails = [ + secretScanGuardrail, + { + name: 'diff-size-cap', + check: async (action: { diff: string }) => { + order.push('guardrail'); + return createDiffSizeGuardrail(500).check(action); + }, + }, + ]; + + await handleFixerPatch(CLEAN_PATCH, { + githubCreatePrTool, + slackTool, + channel: '#incidents', + approvalId: 'approval-3', + head: 'fix/auto', + guardrails: trackedGuardrails as any, + }); + + expect(order).toEqual(['guardrail', 'github']); + }); +}); diff --git a/examples/ops-pipeline/guardedPr.ts b/examples/ops-pipeline/guardedPr.ts new file mode 100644 index 000000000..d96af438f --- /dev/null +++ b/examples/ops-pipeline/guardedPr.ts @@ -0,0 +1,103 @@ +/** + * Ops pipeline: guardrail-gated PR creation wiring (LOU-J7). + * + * After fixer.ts's runFixer() returns {patch}, handleFixerPatch() gates + * that patch through the REAL LOU-E12 runGuardrails() (secret-scan + + * diff-size-cap by default) BEFORE ever considering the GitHub + * PR-creation call. If the guardrail result's `pass` is false, the GitHub + * tool is NEVER called - instead the LOU-J5 Slack tool is called with a + * message naming which guardrail(s) failed and why, reusing the original + * approvalId/channel. If `pass` is true, the REAL LOU-E13/E14-hardened + * GitHub tool's PR-creation function is called exactly once with the + * patch as diff content. + * + * This sequence is synchronous/sequential - the guardrail check is + * `await`ed to completion before the GitHub call is even considered (no + * Promise.all/race with it) - so the guardrail gate is genuinely blocking, + * not parallel or best-effort. Per this epic's own Implementation + * Considerations, this gate is not configurable off: callers may swap + * which guardrails run, but handleFixerPatch() always runs the guardrail + * check before any GitHub call, with no way to skip it. + */ +import { + Guardrail, + ProposedAction, + createDiffSizeGuardrail, + runGuardrails, + secretScanGuardrail, +} from '../../src/execution/guardrails'; +import { ToolDescriptor } from '../../src/types'; + +export interface FixerPatchContext { + /** The REAL LOU-E13/E14-hardened GitHub tool's PR-creation ToolDescriptor. */ + githubCreatePrTool: ToolDescriptor; + /** The LOU-J5 Slack tool used to report a guardrail failure. */ + slackTool: ToolDescriptor; + /** Slack channel to notify on guardrail failure. */ + channel: string; + /** The original approvalId/thread reference, reused for the Slack failure message. */ + approvalId: string; + head: string; + base?: string; + title?: string; + /** + * Guardrails to run before ever considering the GitHub call. Defaults to + * the REAL secretScanGuardrail plus a 500-line createDiffSizeGuardrail - + * both real, non-configurable-off gates per this epic's own + * Implementation Considerations. + */ + guardrails?: Guardrail[]; +} + +export interface HandleFixerPatchResult { + pass: boolean; + failures?: { name: string; reason?: string }[]; + prResult?: unknown; +} + +/** + * Runs the REAL runGuardrails() against `patch` and, ONLY if it passes, + * calls the REAL GitHub PR-creation tool exactly once with the patch as + * diff content (a PR-creation-scoped GitHub token has no endpoint to + * attach a raw diff directly to a PR - see github.ts's scope + * documentation - so the patch is carried in the PR body, the only field + * available to a PR-creation/reading-scoped token that can hold it). + */ +export async function handleFixerPatch( + patch: string, + context: FixerPatchContext +): Promise { + const guardrails = context.guardrails ?? [secretScanGuardrail, createDiffSizeGuardrail(500)]; + const action: ProposedAction = { diff: patch }; + + const guardrailResult = await runGuardrails(action, guardrails); + + if (!guardrailResult.pass) { + const reasons = guardrailResult.failures + .map((f) => `${f.name}${f.reason ? `: ${f.reason}` : ''}`) + .join('; '); + + await context.slackTool.tool.execute!( + { + channel: context.channel, + message: `Guardrail check failed, PR not created (approval ${context.approvalId}): ${reasons}`, + approvalId: context.approvalId, + }, + {} as any + ); + + return { pass: false, failures: guardrailResult.failures }; + } + + const prResult = await context.githubCreatePrTool.tool.execute!( + { + title: context.title ?? `Automated fix (approval ${context.approvalId})`, + body: `Automated fix generated by the ops-pipeline fixer agent.\n\n\`\`\`diff\n${patch}\n\`\`\``, + head: context.head, + base: context.base, + }, + {} as any + ); + + return { pass: true, prResult }; +} diff --git a/examples/ops-pipeline/index.test.ts b/examples/ops-pipeline/index.test.ts new file mode 100644 index 000000000..44dff59c3 --- /dev/null +++ b/examples/ops-pipeline/index.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { startOpsPipeline, OpsPipelineHandle } from './index'; +import { createDemoProvider } from './demoProvider'; +import { createMockGithubTool } from './mocks/mockGithubTool'; +import { createMockSlackTool } from './mocks/mockSlackTool'; +import { createInMemoryApprovalStore } from './slackInteractions'; +import { exampleErrorSignal, sendSyntheticError } from './mocks/mockGrafanaSender'; + +describe('ops-pipeline end-to-end (LOU-J8, against mocks - zero external network)', () => { + let handle: OpsPipelineHandle | undefined; + + afterEach(async () => { + if (handle) { + await handle.close(); + handle = undefined; + } + }); + + it('runs the full pipeline: dedup -> approval gate -> Slack "Fix it" -> guardrail-gated GitHub PR', async () => { + const provider = createDemoProvider(); + const github = createMockGithubTool(); + const slack = createMockSlackTool(); + const approvalStore = createInMemoryApprovalStore(); + + handle = await startOpsPipeline({ + provider, + githubCreatePrTool: github.tool, + slackTool: slack.tool, + approvalStore, + monitorPort: 0, + slackPort: 0, + }); + + const signal = exampleErrorSignal(); + const webhookUrl = `http://127.0.0.1:${handle.monitor.port}/webhook`; + + // New signature is accepted; a duplicate is deduped. + const res1 = await sendSyntheticError(webhookUrl, signal); + expect((await res1.json()).deduped).toBe(false); + const res2 = await sendSyntheticError(webhookUrl, signal); + expect((await res2.json()).deduped).toBe(true); + + // Let the async onResult -> Slack post settle. + await new Promise((resolve) => setTimeout(resolve, 50)); + + // The fixer must NOT have run / no PR yet - blocked behind the approval gate. + expect(slack.posts).toHaveLength(1); + expect(github.createdPullRequests).toHaveLength(0); + + const approvalId = slack.posts[0].approvalId; + expect(approvalId).toBeTruthy(); + + // Simulate the Slack "Fix it" button click. + const interactionsUrl = `http://127.0.0.1:${handle.slack.port}/slack/interactions`; + const interactionRes = await fetch(interactionsUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: approvalId }], + }), + }); + expect((await interactionRes.json()).handled).toBe(true); + + // The approval unblocked the fixer, which produced a patch that passed + // guardrails and resulted in exactly one GitHub PR. + expect(github.createdPullRequests).toHaveLength(1); + expect(github.createdPullRequests[0].body).toContain('OrderService.java'); + }); +}); diff --git a/examples/ops-pipeline/index.ts b/examples/ops-pipeline/index.ts new file mode 100644 index 000000000..029fe6841 --- /dev/null +++ b/examples/ops-pipeline/index.ts @@ -0,0 +1,272 @@ +/** + * Ops pipeline (LOU-J8) - end-to-end demo entry point. + * + * Wires, in order: + * 1. LOU-J4's monitor webhook listener (POST /webhook) + * 2. LOU-J5's Slack tool + interaction route (POST /slack/interactions) + * 3. LOU-J6's fixer, delegated to from the monitor agent + * 4. LOU-J7's guardrail-gated PR creation + * into one process. + * + * Safety property this wiring enforces (per the epic's own Implementation + * Considerations): the fixer agent never runs without first passing + * through the REAL LOU-C approval gate. The `delegate_to_fixer` tool + * registered on the monitor agent is marked `needsApproval: true`, so + * AgentExecutor.execute() pauses and saves a PendingApproval (via the REAL + * ApprovalGate/ApprovalStore) the moment the monitor agent tries to call + * it - the fixer agent's AgentExecutor.execute() call is only ever reached + * from inside that tool's execute(), which only runs after a human clicks + * "Fix it" in Slack and resumeAfterApproval() (LOU-C6, called for real by + * slackInteractions.ts) resolves the approval and runs the deferred tool + * call. There is no code path here that runs the fixer without that + * approval round-trip. + * + * Defaults to the mocks/ implementations (zero external network access); + * pass real tool descriptors/provider to wire it to a real Slack webhook, + * GitHub token and LLM provider instead. + */ +import * as http from 'node:http'; +import { AgentType, ToolDescriptor } from '../../src/types'; +import { ToolRegistry } from '../../src/tools'; +import { LLMProvider } from '../../src/providers/llm'; +import { startMonitorServer, MonitorServerHandle } from './monitor'; +import { createFixerDelegateTool, buildFixerAgent } from './fixer'; +import { handleFixerPatch } from './guardedPr'; +import { + createInMemoryApprovalStore, + extractApprovalIdFromInteraction, + SlackInteractionPayload, + handleSlackInteraction, +} from './slackInteractions'; +import { ApprovalStore } from '../../src/execution/ApprovalGate'; +import { ExecutionResult } from '../../src/execution/AgentExecutor'; +import { Guardrail } from '../../src/execution/guardrails'; +import { createDemoProvider } from './demoProvider'; +import { createMockGithubTool } from './mocks/mockGithubTool'; +import { createMockSlackTool } from './mocks/mockSlackTool'; + +const DELEGATE_TOOL_NAME = 'delegate_to_fixer'; +const DEFAULT_CHANNEL = '#incidents'; + +/** + * Pulls the fixer's extracted `patch` back out of a resumed ExecutionResult + * by finding the delegate tool's own result message (AgentExecutor appends + * it as a `role: 'tool'` message whose content is + * `JSON.stringify(toolResult.result)`, and createFixerDelegateTool's + * result shape includes a `patch` field - see fixer.ts). + */ +function extractDelegatedPatch(result: ExecutionResult, toolName: string): string | undefined { + for (let i = result.messages.length - 1; i >= 0; i--) { + const message = result.messages[i]; + if (message.role === 'tool' && message.toolName === toolName) { + try { + const parsed = JSON.parse(message.content); + if (typeof parsed?.patch === 'string') { + return parsed.patch; + } + } catch { + // Not JSON / no patch field - keep scanning older messages. + } + } + } + return undefined; +} + +export interface OpsPipelineDeps { + provider?: LLMProvider; + githubCreatePrTool?: ToolDescriptor; + slackTool?: ToolDescriptor; + approvalStore?: ApprovalStore; + monitorHost?: string; + monitorPort?: number; + slackHost?: string; + slackPort?: number; + channel?: string; + /** + * Guardrails to run before considering the GitHub PR call. Defaults to + * handleFixerPatch()'s own defaults (real secretScanGuardrail + a + * diff-size cap) when omitted. Exposed here so callers (e.g. LOU-J9's + * pipeline.eval.ts) can wrap the default guardrails to observe/record + * execution order without changing which guardrails actually run. + */ + guardrails?: Guardrail[]; +} + +export interface OpsPipelineHandle { + monitor: MonitorServerHandle; + slack: { server: http.Server; port: number; close: () => Promise }; + close: () => Promise; +} + +const MAX_BODY_BYTES = 1024 * 1024; + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + let bytes = 0; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + reject(new Error(`Request body exceeds ${MAX_BODY_BYTES} byte limit`)); + req.destroy(); + return; + } + body += chunk; + }); + req.on('end', () => resolve(body)); + req.on('error', reject); + }); +} + +function parseSlackInteractionBody(raw: string): SlackInteractionPayload { + const trimmed = raw.trim(); + if (trimmed.startsWith('{')) { + return JSON.parse(trimmed); + } + const params = new URLSearchParams(raw); + const payloadField = params.get('payload'); + if (!payloadField) { + throw new Error('Slack interaction body missing "payload" field'); + } + return JSON.parse(payloadField); +} + +/** + * Starts the full ops pipeline: the monitor webhook server, and a second + * small server exposing POST /slack/interactions (kept as a separate + * listener from the monitor server rather than merged into one + * http.Server, since startMonitorServer() already owns its own server + * lifecycle from LOU-J4 - both are still started from this one process + * entry point). + */ +export async function startOpsPipeline(deps: OpsPipelineDeps = {}): Promise { + const provider = deps.provider ?? createDemoProvider(); + const githubCreatePrTool = deps.githubCreatePrTool ?? createMockGithubTool().tool; + const slackTool = deps.slackTool ?? createMockSlackTool().tool; + const approvalStore = deps.approvalStore ?? createInMemoryApprovalStore(); + const channel = deps.channel ?? DEFAULT_CHANNEL; + + const fixerAgent = buildFixerAgent(); + const delegateTool = createFixerDelegateTool({ agent: fixerAgent, provider }); + // The fixer agent must never run without first passing through the + // approval gate (this epic's own non-negotiable safety property) - + // enforced here by flagging the delegate tool itself as needing + // approval, so AgentExecutor.execute() pauses BEFORE ever calling it. + const gatedDelegateTool: ToolDescriptor = { ...delegateTool, needsApproval: true }; + + const toolRegistry = new ToolRegistry(); + toolRegistry.register(DELEGATE_TOOL_NAME, gatedDelegateTool); + + const monitorAgent = { + name: 'ops-monitor', + agentType: AgentType.SmartAssistant, + prompt: + 'You are an ops monitor. When an alert fires, delegate it to the fixer agent via the ' + + `${DELEGATE_TOOL_NAME} tool.`, + tools: { [DELEGATE_TOOL_NAME]: { tool: DELEGATE_TOOL_NAME } }, + }; + + const monitor = await startMonitorServer({ + executeOptions: { + agent: monitorAgent, + provider, + toolRegistry, + approvalStore, + }, + host: deps.monitorHost, + port: deps.monitorPort, + onResult: async (signal, result) => { + if (result.finishReason === 'awaiting-approval' && result.approvalId) { + await slackTool.tool.execute!( + { + channel, + message: `New error detected (${signal.signature}): ${signal.message}. Approve the fix?`, + approvalId: result.approvalId, + }, + {} as any + ); + } + }, + }); + + const slackServer = http.createServer((req, res) => { + void (async () => { + if (req.method === 'POST' && req.url === '/slack/interactions') { + try { + const raw = await readBody(req); + const payload = parseSlackInteractionBody(raw); + const approvalId = extractApprovalIdFromInteraction(payload); + + const result = await handleSlackInteraction(payload, { + approvalStore, + toolRegistry, + provider, + }); + + if (result && approvalId) { + const patch = extractDelegatedPatch(result, DELEGATE_TOOL_NAME); + if (patch) { + await handleFixerPatch(patch, { + githubCreatePrTool, + slackTool, + channel, + approvalId, + head: `fix/auto-${approvalId}`, + guardrails: deps.guardrails, + }); + } + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ handled: result !== undefined })); + } catch (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + return; + } + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + })(); + }); + + const slackHost = deps.slackHost ?? '127.0.0.1'; + const slackPort = deps.slackPort ?? 0; + await new Promise((resolve, reject) => { + slackServer.once('error', reject); + slackServer.listen(slackPort, slackHost, () => resolve()); + }); + const slackAddress = slackServer.address(); + const actualSlackPort = typeof slackAddress === 'object' && slackAddress ? slackAddress.port : slackPort; + + return { + monitor, + slack: { + server: slackServer, + port: actualSlackPort, + close: () => new Promise((resolve, reject) => slackServer.close((e) => (e ? reject(e) : resolve()))), + }, + close: async () => { + await monitor.close(); + await new Promise((resolve, reject) => slackServer.close((e) => (e ? reject(e) : resolve()))); + }, + }; +} + +if (require.main === module) { + const monitorPort = Number(process.env.OPS_PIPELINE_MONITOR_PORT) || 8787; + const slackPort = Number(process.env.OPS_PIPELINE_SLACK_PORT) || 8788; + + startOpsPipeline({ monitorPort, slackPort }) + .then((handle) => { + console.log(`ops-pipeline monitor webhook listening on http://127.0.0.1:${handle.monitor.port}/webhook`); + console.log( + `ops-pipeline slack interactions listening on http://127.0.0.1:${handle.slack.port}/slack/interactions` + ); + console.log('Running against mock Slack/GitHub APIs and a deterministic demo provider by default.'); + }) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/examples/ops-pipeline/mocks/mockGithubTool.ts b/examples/ops-pipeline/mocks/mockGithubTool.ts new file mode 100644 index 000000000..90ddfc876 --- /dev/null +++ b/examples/ops-pipeline/mocks/mockGithubTool.ts @@ -0,0 +1,48 @@ +/** + * Mock GitHub API (LOU-J8) - same call shape as the REAL github tool's + * `github_create_pull_request` (src/tools/built-in/github.ts): takes + * {title, body, head, base}, returns JSON matching {number, url, state}. + * Never makes a real network call, so the ops-pipeline demo runs with zero + * external network access. + */ +import { ToolDescriptor } from '../../../src/types'; + +export interface MockCreatedPullRequest { + title: string; + body: string; + head: string; + base?: string; +} + +export interface MockGithubTool { + tool: ToolDescriptor; + createdPullRequests: MockCreatedPullRequest[]; +} + +export function createMockGithubTool(): MockGithubTool { + const createdPullRequests: MockCreatedPullRequest[] = []; + let nextNumber = 1; + + const tool: ToolDescriptor = { + displayName: 'Create GitHub Pull Request (mock)', + tool: { + description: 'Mock: creates a pull request', + parameters: {} as any, + execute: async (args: MockCreatedPullRequest) => { + createdPullRequests.push(args); + const number = nextNumber++; + return JSON.stringify( + { + number, + url: `https://github.com/mock-org/mock-repo/pull/${number}`, + state: 'open', + }, + null, + 2 + ); + }, + } as any, + }; + + return { tool, createdPullRequests }; +} diff --git a/examples/ops-pipeline/mocks/mockGrafanaSender.ts b/examples/ops-pipeline/mocks/mockGrafanaSender.ts new file mode 100644 index 000000000..fc759a49e --- /dev/null +++ b/examples/ops-pipeline/mocks/mockGrafanaSender.ts @@ -0,0 +1,30 @@ +/** + * Mock Grafana/Datadog webhook sender (LOU-J8) - same call shape as a real + * monitoring source: a plain HTTP POST of an ErrorSignal JSON body to the + * pipeline's /webhook endpoint. Used by the demo's "post a synthetic + * error" script; posts only to the demo's own localhost server, so this + * never reaches the real network either. + */ +import { ErrorSignal } from '../monitor'; + +export async function sendSyntheticError(webhookUrl: string, signal: ErrorSignal): Promise { + return fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(signal), + }); +} + +/** A representative synthetic error signal for the demo. */ +export function exampleErrorSignal(): ErrorSignal { + return { + signature: 'demo-npe-order-service', + service: 'grafana', + message: 'NullPointerException in OrderService.charge', + logs: [ + 'java.lang.NullPointerException', + '\tat OrderService.charge(OrderService.java:42)', + '\tat OrderController.checkout(OrderController.java:18)', + ].join('\n'), + }; +} diff --git a/examples/ops-pipeline/mocks/mockSlackTool.ts b/examples/ops-pipeline/mocks/mockSlackTool.ts new file mode 100644 index 000000000..c7075c74b --- /dev/null +++ b/examples/ops-pipeline/mocks/mockSlackTool.ts @@ -0,0 +1,54 @@ +/** + * Mock Slack API (LOU-J8) - same call shape as the REAL Slack tool + * (src/tools/built-in/slack.ts's createSlackTool()): takes + * {channel, message, approvalId}, builds the exact same Block Kit payload + * via the real buildSlackAlertPayload(), and records it instead of + * POSTing to a real webhook - so the ops-pipeline demo runs with zero + * external network access. + */ +import { ToolDescriptor } from '../../../src/types'; +import { buildSlackAlertPayload, SlackAlertPayload } from '../../../src/tools/built-in/slack'; + +export interface MockSlackPost { + channel: string; + message: string; + approvalId: string; + payload: SlackAlertPayload; +} + +export interface MockSlackTool { + tool: ToolDescriptor; + posts: MockSlackPost[]; +} + +export function createMockSlackTool(): MockSlackTool { + const posts: MockSlackPost[] = []; + + const tool: ToolDescriptor = { + displayName: 'Post Slack Alert (mock)', + tool: { + description: 'Mock: posts a Slack alert with a Fix it button', + parameters: {} as any, + execute: async ({ + channel, + message, + approvalId, + }: { + channel: string; + message: string; + approvalId: string; + }) => { + const payload = buildSlackAlertPayload(channel, message, approvalId); + posts.push({ channel, message, approvalId, payload }); + // Logged so the README's manual "click Fix it" step is actually + // followable when running the live demo from the terminal - the + // mock has no real Slack UI to click a button in. + console.log(`[mock-slack] alert posted to ${channel} — approvalId: ${approvalId}`); + console.log(`[mock-slack] ${message}`); + return { ok: true }; + }, + } as any, + }; + + return { tool, posts }; +} diff --git a/examples/ops-pipeline/mocks/sendSyntheticError.ts b/examples/ops-pipeline/mocks/sendSyntheticError.ts new file mode 100644 index 000000000..6f2bb2de0 --- /dev/null +++ b/examples/ops-pipeline/mocks/sendSyntheticError.ts @@ -0,0 +1,24 @@ +/** + * Standalone script: POSTs a synthetic ErrorSignal at the running + * ops-pipeline demo's /webhook endpoint (LOU-J8 README step). + * + * Usage: tsx examples/ops-pipeline/mocks/sendSyntheticError.ts [port] + * (defaults to port 8787, matching index.ts's default monitor port) + */ +import { sendSyntheticError, exampleErrorSignal } from './mockGrafanaSender'; + +async function main() { + const port = Number(process.argv[2]) || 8787; + const url = `http://127.0.0.1:${port}/webhook`; + const signal = exampleErrorSignal(); + + console.log(`POSTing synthetic error signature "${signal.signature}" to ${url} ...`); + const response = await sendSyntheticError(url, signal); + const body = await response.json(); + console.log(`Response ${response.status}:`, body); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/examples/ops-pipeline/monitor.test.ts b/examples/ops-pipeline/monitor.test.ts new file mode 100644 index 000000000..40e1da98b --- /dev/null +++ b/examples/ops-pipeline/monitor.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, vi } from 'vitest'; +import { handleErrorSignal, buildMonitorPrompt, ErrorSignal, startMonitorServer } from './monitor'; +import { AgentExecutor } from '../../src/execution/AgentExecutor'; +import { AgentType } from '../../src/types'; +import { createMockProvider } from '../../src/providers/mock'; + +function makeSignal(overrides: Partial = {}): ErrorSignal { + return { + signature: 'sig-1', + service: 'grafana', + message: 'NullPointerException in OrderService', + logs: 'at OrderService.charge(OrderService.java:42)', + ...overrides, + }; +} + +const executeOptions = { + agent: { name: 'monitor', agentType: AgentType.SmartAssistant, prompt: 'You are the monitor.' }, + provider: createMockProvider({ responses: ['ack'] }), +}; + +describe('buildMonitorPrompt', () => { + it('includes the signature, message and logs', () => { + const prompt = buildMonitorPrompt(makeSignal()); + expect(prompt).toContain('sig-1'); + expect(prompt).toContain('NullPointerException in OrderService'); + expect(prompt).toContain('OrderService.java:42'); + }); +}); + +describe('handleErrorSignal', () => { + it('calls AgentExecutor.execute() exactly once for a new signature', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + + const result = await handleErrorSignal(makeSignal(), executeOptions, seen); + + expect(spy).toHaveBeenCalledTimes(1); + expect(result).toBeDefined(); + expect(spy.mock.calls[0][0].input).toContain('sig-1'); + + spy.mockRestore(); + }); + + it('does not call the executor again for a duplicate signature', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + + await handleErrorSignal(makeSignal(), executeOptions, seen); + expect(spy).toHaveBeenCalledTimes(1); + + const secondResult = await handleErrorSignal(makeSignal(), executeOptions, seen); + expect(spy).toHaveBeenCalledTimes(1); // zero additional calls + expect(secondResult).toBeUndefined(); + + spy.mockRestore(); + }); + + it('treats different signatures as independent (executor called once each)', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + + await handleErrorSignal(makeSignal({ signature: 'sig-a' }), executeOptions, seen); + await handleErrorSignal(makeSignal({ signature: 'sig-b' }), executeOptions, seen); + + expect(spy).toHaveBeenCalledTimes(2); + spy.mockRestore(); + }); +}); + +describe('startMonitorServer', () => { + it('accepts a POST /webhook and dedupes by signature over HTTP', async () => { + const spy = vi.spyOn(AgentExecutor, 'execute'); + const seen = new Set(); + const handle = await startMonitorServer({ executeOptions, seen, port: 0 }); + + try { + const url = `http://127.0.0.1:${handle.port}/webhook`; + const signal = makeSignal({ signature: 'http-sig' }); + + const res1 = await fetch(url, { method: 'POST', body: JSON.stringify(signal) }); + const body1 = await res1.json(); + expect(res1.status).toBe(202); + expect(body1.deduped).toBe(false); + + const res2 = await fetch(url, { method: 'POST', body: JSON.stringify(signal) }); + const body2 = await res2.json(); + expect(body2.deduped).toBe(true); + + expect(spy).toHaveBeenCalledTimes(1); + } finally { + spy.mockRestore(); + await handle.close(); + } + }); +}); diff --git a/examples/ops-pipeline/monitor.ts b/examples/ops-pipeline/monitor.ts new file mode 100644 index 000000000..decf940a9 --- /dev/null +++ b/examples/ops-pipeline/monitor.ts @@ -0,0 +1,200 @@ +/** + * Ops pipeline: Grafana/Datadog error-signal monitor (LOU-J4). + * + * Receives an ErrorSignal (from a Grafana or Datadog alert webhook, or from + * the mock sender in mocks/ for the demo), dedupes it by `signature` so a + * flapping/retried alert never triggers more than one agent run, and - on a + * genuinely new signature - calls the REAL static AgentExecutor.execute() + * (src/execution/AgentExecutor.ts) exactly once with a prompt built by + * buildMonitorPrompt(). + * + * A small HTTP listener (startMonitorServer) exposes this as `POST + * /webhook`, reusing the same http.createServer pattern already + * established in src/cli/dev.ts: bound to localhost by default, and a + * request-body size cap so an oversized payload can't exhaust memory. + */ +import * as http from 'node:http'; +import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../../src/execution/AgentExecutor'; + +/** + * A single error/incident signal from a monitoring source. + */ +export interface ErrorSignal { + /** Stable identity for this specific error condition, used for dedup. */ + signature: string; + service: 'grafana' | 'datadog'; + message: string; + logs: string; +} + +/** + * Everything AgentExecutor.execute() needs EXCEPT `input` - handleErrorSignal + * builds `input` itself from the signal via buildMonitorPrompt(). + */ +export type MonitorExecuteOptions = Omit; + +/** + * Module-level default dedup set, used when handleErrorSignal() is called + * without an explicit `seen` set (e.g. from startMonitorServer(), where one + * long-lived set should track dedup for the life of the process). Tests + * pass their own Set so cases stay isolated from each other and from this + * module-level state. + */ +const defaultSeenSignatures = new Set(); + +/** + * Builds the prompt sent to the monitor agent for a given signal. + */ +export function buildMonitorPrompt(signal: ErrorSignal): string { + return [ + `A ${signal.service} alert fired with signature "${signal.signature}".`, + `Message: ${signal.message}`, + '', + 'Logs:', + signal.logs, + '', + 'Diagnose this and delegate a fix request to the fixer agent if one is available.', + ].join('\n'); +} + +/** + * Handles one incoming ErrorSignal: deduped by `signal.signature` via an + * in-memory Set - if this signature has already been seen, returns + * immediately without calling the executor at all. Otherwise records the + * signature and calls the real AgentExecutor.execute() exactly once. + */ +export async function handleErrorSignal( + signal: ErrorSignal, + executeOptions: MonitorExecuteOptions, + seen: Set = defaultSeenSignatures +): Promise { + if (seen.has(signal.signature)) { + return undefined; + } + seen.add(signal.signature); + + return AgentExecutor.execute({ + ...executeOptions, + input: buildMonitorPrompt(signal), + }); +} + +/** Body-size cap for POST /webhook, matching src/cli/dev.ts's /chat convention. */ +const MAX_BODY_BYTES = 1024 * 1024; // 1MB + +class PayloadTooLargeError extends Error {} + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + let bytes = 0; + let tooLarge = false; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + tooLarge = true; + return; + } + body += chunk; + }); + req.on('end', () => { + if (tooLarge) { + reject(new PayloadTooLargeError(`Request body exceeds ${MAX_BODY_BYTES} byte limit`)); + return; + } + resolve(body); + }); + req.on('error', reject); + }); +} + +export interface MonitorServerHandle { + server: http.Server; + port: number; + close: () => Promise; +} + +export interface StartMonitorServerOptions { + executeOptions: MonitorExecuteOptions; + host?: string; + port?: number; + seen?: Set; + /** Called after each handled (non-duplicate) signal, mainly for tests/demo logging. */ + onResult?: (signal: ErrorSignal, result: ExecutionResult) => void; +} + +/** + * Starts an HTTP server exposing `POST /webhook` for ErrorSignal payloads. + * Bound to '127.0.0.1' by default (localhost-only), matching src/cli/dev.ts's + * security posture for local tooling - pass an explicit host to opt in to + * wider access. + */ +export async function startMonitorServer( + options: StartMonitorServerOptions +): Promise { + const host = options.host ?? '127.0.0.1'; + const port = options.port ?? 0; + const seen = options.seen ?? defaultSeenSignatures; + + const server = http.createServer((req, res) => { + void (async () => { + if (req.method === 'POST' && req.url === '/webhook') { + try { + const body = await readBody(req); + const signal = JSON.parse(body || '{}') as Partial; + if ( + typeof signal.signature !== 'string' || + typeof signal.service !== 'string' || + typeof signal.message !== 'string' || + typeof signal.logs !== 'string' + ) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + error: 'Request body must be an ErrorSignal {signature, service, message, logs}', + }) + ); + return; + } + + const result = await handleErrorSignal(signal as ErrorSignal, options.executeOptions, seen); + if (result && options.onResult) { + options.onResult(signal as ErrorSignal, result); + } + + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ deduped: result === undefined })); + } catch (error) { + if (error instanceof PayloadTooLargeError) { + res.writeHead(413, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: error.message })); + return; + } + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + return; + } + + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + })(); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, () => resolve()); + }); + + const address = server.address(); + const actualPort = typeof address === 'object' && address ? address.port : port; + + return { + server, + port: actualPort, + close: () => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }), + }; +} diff --git a/examples/ops-pipeline/pipeline.eval.ts b/examples/ops-pipeline/pipeline.eval.ts new file mode 100644 index 000000000..fc9842ed8 --- /dev/null +++ b/examples/ops-pipeline/pipeline.eval.ts @@ -0,0 +1,172 @@ +/** + * Ops pipeline eval (LOU-J9). + * + * Two complementary checks: + * + * 1. A REAL defineEval() (LOU-G1) using the REAL toolCallOrder() scorer + * (LOU-G4) against the monitor agent's own single + * AgentExecutor.execute() step. This step's ExecutionResult.toolCalls + * DOES naturally carry the expected sequence (a single delegate call to + * the real `delegate_to_fixer` tool name used by index.ts), so + * toolCallOrder() applies directly and honestly here. + * + * 2. The ops-pipeline as a whole is composed of multiple SEPARATE tool + * calls across monitor -> Slack alert -> (approval) -> fixer delegate + * -> guardrails -> GitHub PR creation, most of which do NOT flow + * through one single AgentExecutor.execute() call's `toolCalls` field + * (the Slack/GitHub calls are plain ToolDescriptor invocations made + * directly by index.ts/guardedPr.ts, not LLM-issued tool calls). A + * defineEval()+toolCallOrder() call cannot express that cross-stage + * sequence, so - per this ticket's own guidance - it's instead recorded + * directly: every real stage (the monitor's delegate tool call, the + * Slack alert, both real guardrails, and GitHub PR creation) is + * instrumented with the same spy pattern fixer.test.ts/guardedPr.test.ts/ + * index.test.ts already use, and the recorded order is asserted with a + * plain deep-equals - the most honest representation of what actually + * ran, in what order, without inventing a fake single-result shape. + */ +import { describe, it, expect } from 'vitest'; +import { defineEval } from '../../src/evals/defineEval'; +import { toolCallOrder } from '../../src/evals/scorers'; +import { AgentType } from '../../src/types'; +import { ToolRegistry } from '../../src/tools'; +import { secretScanGuardrail, createDiffSizeGuardrail } from '../../src/execution/guardrails'; +import { startOpsPipeline } from './index'; +import { createDemoProvider } from './demoProvider'; +import { createFixerDelegateTool, buildFixerAgent } from './fixer'; +import { buildMonitorPrompt } from './monitor'; +import { createMockGithubTool } from './mocks/mockGithubTool'; +import { createMockSlackTool } from './mocks/mockSlackTool'; +import { createInMemoryApprovalStore } from './slackInteractions'; +import { exampleErrorSignal, sendSyntheticError } from './mocks/mockGrafanaSender'; + +const DELEGATE_TOOL_NAME = 'delegate_to_fixer'; +const FIXTURE_SIGNAL = exampleErrorSignal(); + +// --------------------------------------------------------------------------- +// 1. REAL defineEval() + REAL toolCallOrder() scorer (LOU-G1/G4) +// --------------------------------------------------------------------------- + +const evalProvider = createDemoProvider(); +const evalFixerAgent = buildFixerAgent(); +const evalDelegateTool = createFixerDelegateTool({ agent: evalFixerAgent, provider: evalProvider }); +// Note: unlike index.ts's real wiring, this eval registry does NOT flag the +// delegate tool needsApproval:true - defineEval()'s EvalConfig (LOU-G1) has +// no approvalStore field to satisfy a paused run, and the approval-gate +// property itself is already covered directly by index.test.ts. This eval +// is purely about the monitor agent's own tool-call CHOICE/ORDER. +const evalToolRegistry = new ToolRegistry(); +evalToolRegistry.register(DELEGATE_TOOL_NAME, evalDelegateTool); + +const evalMonitorAgent = { + name: 'ops-monitor', + agentType: AgentType.SmartAssistant, + prompt: `You are an ops monitor. When an alert fires, delegate it to the fixer agent via the ${DELEGATE_TOOL_NAME} tool.`, + tools: { [DELEGATE_TOOL_NAME]: { tool: DELEGATE_TOOL_NAME } }, +}; + +defineEval({ + name: 'ops-pipeline: monitor triages a new error by calling delegate_to_fixer exactly once', + agent: evalMonitorAgent, + input: buildMonitorPrompt(FIXTURE_SIGNAL), + provider: evalProvider, + toolRegistry: evalToolRegistry, + score: toolCallOrder([{ tool: DELEGATE_TOOL_NAME }]), + threshold: 1, +}); + +// --------------------------------------------------------------------------- +// 2. Full cross-stage call-order recording (direct assertion) +// --------------------------------------------------------------------------- + +describe('ops-pipeline: full cross-stage call order (LOU-J9)', () => { + it('runs monitor -> slack alert -> fixer delegate -> guardrails -> github PR, in that order', async () => { + const order: string[] = []; + + const provider = createDemoProvider(); + const approvalStore = createInMemoryApprovalStore(); + + const github = createMockGithubTool(); + const slack = createMockSlackTool(); + + // Wrap the mock tools' execute() to record call order without changing + // their real behavior/return shape. + const githubCreatePrTool = { + ...github.tool, + tool: { + ...github.tool.tool, + execute: async (...args: Parameters>) => { + order.push('github:create_pull_request'); + return github.tool.tool.execute!(...args); + }, + }, + }; + const slackTool = { + ...slack.tool, + tool: { + ...slack.tool.tool, + execute: async (...args: Parameters>) => { + order.push('slack:alert'); + return slack.tool.tool.execute!(...args); + }, + }, + }; + + const trackedGuardrails = [ + { + name: 'secret-scan', + check: async (action: { diff: string }) => { + order.push('guardrail:secret-scan'); + return secretScanGuardrail.check(action); + }, + }, + { + name: 'diff-size-cap', + check: async (action: { diff: string }) => { + order.push('guardrail:diff-size-cap'); + return createDiffSizeGuardrail(500).check(action); + }, + }, + ]; + + const handle = await startOpsPipeline({ + provider, + githubCreatePrTool: githubCreatePrTool as any, + slackTool: slackTool as any, + approvalStore, + guardrails: trackedGuardrails as any, + monitorPort: 0, + slackPort: 0, + }); + + try { + const webhookUrl = `http://127.0.0.1:${handle.monitor.port}/webhook`; + await sendSyntheticError(webhookUrl, FIXTURE_SIGNAL); + await new Promise((resolve) => setTimeout(resolve, 50)); // let the monitor's onResult -> Slack alert settle + + const approvalId = slack.posts[0]?.approvalId; + expect(approvalId).toBeTruthy(); + + const interactionsUrl = `http://127.0.0.1:${handle.slack.port}/slack/interactions`; + await fetch(interactionsUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: approvalId }], + }), + }); + + const EXPECTED_ORDER = [ + 'slack:alert', + 'guardrail:secret-scan', + 'guardrail:diff-size-cap', + 'github:create_pull_request', + ]; + + expect(order).toEqual(EXPECTED_ORDER); + } finally { + await handle.close(); + } + }); +}); diff --git a/examples/ops-pipeline/slackInteractions.test.ts b/examples/ops-pipeline/slackInteractions.test.ts new file mode 100644 index 000000000..a53182344 --- /dev/null +++ b/examples/ops-pipeline/slackInteractions.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import * as resumeModule from '../../src/execution/resume'; +import { + extractApprovalIdFromInteraction, + handleSlackInteraction, + createInMemoryApprovalStore, + SlackInteractionPayload, +} from './slackInteractions'; +import { ApprovalStore, ExecutionSnapshot, PendingApproval } from '../../src/execution/ApprovalGate'; +import { ToolRegistry } from '../../src/tools'; +import { createMockProvider } from '../../src/providers/mock'; +import { AgentType } from '../../src/types'; + +describe('extractApprovalIdFromInteraction', () => { + it('extracts the approvalId from a fix_it block_actions click', () => { + const payload: SlackInteractionPayload = { + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: 'approval-abc' }], + }; + expect(extractApprovalIdFromInteraction(payload)).toBe('approval-abc'); + }); + + it('returns undefined for a non-fix_it action', () => { + const payload: SlackInteractionPayload = { + type: 'block_actions', + actions: [{ action_id: 'something_else', value: 'x' }], + }; + expect(extractApprovalIdFromInteraction(payload)).toBeUndefined(); + }); + + it('returns undefined for a non-block_actions payload', () => { + expect(extractApprovalIdFromInteraction({ type: 'view_submission' })).toBeUndefined(); + }); +}); + +describe('handleSlackInteraction routes to the REAL resumeAfterApproval', () => { + let approvalStore: ApprovalStore; + let toolRegistry: ToolRegistry; + const provider = createMockProvider({ responses: ['done'] }); + + beforeEach(() => { + approvalStore = createInMemoryApprovalStore(); + toolRegistry = new ToolRegistry(); + }); + + it('calls resumeAfterApproval exactly once with the matching id, and it actually resolves the pending approval', async () => { + const resumeSpy = vi.spyOn(resumeModule, 'resumeAfterApproval'); + + const execute = vi.fn().mockResolvedValue({ fixed: true }); + toolRegistry.register('apply_fix', { + displayName: 'Apply Fix', + tool: { description: 'apply fix', parameters: {}, execute } as any, + needsApproval: true, + }); + + const pending: PendingApproval = { + id: 'approval-abc', + toolCallId: 'call-1', + toolName: 'apply_fix', + args: {}, + createdAt: new Date().toISOString(), + }; + const snapshot: ExecutionSnapshot = { + agent: { name: 'fixer', agentType: AgentType.SmartAssistant, prompt: 'fix it' }, + currentMessages: [], + pendingToolCall: pending, + steps: 1, + }; + await approvalStore.save(pending, snapshot); + + const payload: SlackInteractionPayload = { + type: 'block_actions', + actions: [{ action_id: 'fix_it', value: 'approval-abc' }], + }; + + const result = await handleSlackInteraction(payload, { approvalStore, toolRegistry, provider }); + + expect(resumeSpy).toHaveBeenCalledTimes(1); + expect(resumeSpy.mock.calls[0][0]).toEqual({ id: 'approval-abc', approved: true }); + expect(execute).toHaveBeenCalledTimes(1); // the deferred tool actually ran + expect(result).toBeDefined(); + + resumeSpy.mockRestore(); + }); + + it('does not call resumeAfterApproval for a non-fix_it payload', async () => { + const resumeSpy = vi.spyOn(resumeModule, 'resumeAfterApproval'); + + const payload: SlackInteractionPayload = { type: 'view_submission' }; + const result = await handleSlackInteraction(payload, { approvalStore, toolRegistry, provider }); + + expect(resumeSpy).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); + + resumeSpy.mockRestore(); + }); +}); diff --git a/examples/ops-pipeline/slackInteractions.ts b/examples/ops-pipeline/slackInteractions.ts new file mode 100644 index 000000000..5665dbdba --- /dev/null +++ b/examples/ops-pipeline/slackInteractions.ts @@ -0,0 +1,151 @@ +/** + * Slack interaction-callback route (LOU-J5). + * + * Parses a Slack interaction callback (what Slack POSTs to the + * `POST /slack/interactions` endpoint when a user clicks the "Fix it" + * button built by src/tools/built-in/slack.ts's buildSlackAlertPayload()), + * extracts the approvalId from the clicked button's `value`, and calls the + * REAL LOU-C6 `resumeAfterApproval()` (src/execution/resume.ts) with that + * id - genuinely resuming the paused AgentExecutor run through the real + * ApprovalGate, not a bypass or a reimplementation of it. + */ +import * as http from 'node:http'; +import { + resumeAfterApproval, + ResumeExecuteOptions, +} from '../../src/execution/resume'; +import { ApprovalStore, ExecutionSnapshot, PendingApproval } from '../../src/execution/ApprovalGate'; +import { ExecutionResult } from '../../src/execution/AgentExecutor'; +import { LLMProvider } from '../../src/providers'; +import { ToolRegistry } from '../../src/tools'; + +/** The subset of Slack's real block_actions interaction callback payload this route reads. */ +export interface SlackInteractionPayload { + type: string; + actions?: Array<{ action_id: string; value?: string }>; +} + +/** + * Extracts the approvalId (the clicked button's `value`) from a Slack + * `block_actions` interaction payload whose action_id is 'fix_it'. Returns + * undefined if the payload isn't a recognized fix_it click. + */ +export function extractApprovalIdFromInteraction(payload: SlackInteractionPayload): string | undefined { + if (payload.type !== 'block_actions' || !payload.actions) { + return undefined; + } + const fixItAction = payload.actions.find((a) => a.action_id === 'fix_it'); + return fixItAction?.value; +} + +/** + * A simple in-memory ApprovalStore, suitable for the ops-pipeline demo and + * for tests. Mirrors the pattern src/execution/resume.test.ts uses for its + * own in-memory store. + */ +export function createInMemoryApprovalStore(): ApprovalStore { + const records = new Map(); + return { + async save(pending, snapshot) { + records.set(pending.id, { pending, snapshot }); + }, + async resolve(id) { + const record = records.get(id); + if (!record) return null; + records.delete(id); + return record; + }, + }; +} + +export interface SlackInteractionsDeps { + approvalStore: ApprovalStore; + toolRegistry: ToolRegistry; + provider: LLMProvider; + executeOptions?: ResumeExecuteOptions; +} + +/** + * Handles one parsed Slack interaction payload: extracts the approvalId and + * resumes the paused run via the REAL resumeAfterApproval(), approving the + * deferred tool call (clicking "Fix it" is an approval, not a rejection). + * Returns undefined (without calling resumeAfterApproval at all) if the + * payload isn't a recognized fix_it click. + */ +export async function handleSlackInteraction( + payload: SlackInteractionPayload, + deps: SlackInteractionsDeps +): Promise { + const approvalId = extractApprovalIdFromInteraction(payload); + if (!approvalId) { + return undefined; + } + + return resumeAfterApproval( + { id: approvalId, approved: true }, + deps.approvalStore, + deps.toolRegistry, + deps.provider, + deps.executeOptions ?? {} + ); +} + +const MAX_BODY_BYTES = 1024 * 1024; // 1MB + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + let bytes = 0; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + reject(new Error(`Request body exceeds ${MAX_BODY_BYTES} byte limit`)); + req.destroy(); + return; + } + body += chunk; + }); + req.on('end', () => resolve(body)); + req.on('error', reject); + }); +} + +/** + * Slack sends interaction callbacks as `application/x-www-form-urlencoded` + * with the JSON payload under the `payload` field - this parses either that + * form, or a raw JSON body (for convenience/tests). + */ +function parseSlackInteractionBody(raw: string): SlackInteractionPayload { + const trimmed = raw.trim(); + if (trimmed.startsWith('{')) { + return JSON.parse(trimmed); + } + const params = new URLSearchParams(raw); + const payloadField = params.get('payload'); + if (!payloadField) { + throw new Error('Slack interaction body missing "payload" field'); + } + return JSON.parse(payloadField); +} + +/** + * Builds an http.RequestListener-compatible handler for `POST + * /slack/interactions`, for use with node:http's createServer (see + * examples/ops-pipeline/index.ts). + */ +export function createSlackInteractionsRoute( + deps: SlackInteractionsDeps +): (req: http.IncomingMessage, res: http.ServerResponse) => Promise { + return async (req, res) => { + try { + const raw = await readBody(req); + const payload = parseSlackInteractionBody(raw); + const result = await handleSlackInteraction(payload, deps); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ handled: result !== undefined })); + } catch (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + }; +} diff --git a/package.json b/package.json index d3a9e5943..83b23b615 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,8 @@ "example:workflow-router": "tsx examples/workflow-router/index.ts", "example:doc-qa": "tsx examples/doc-qa/index.ts", "example:slack-notifier": "tsx examples/slack-notifier/index.ts", + "pipeline:demo": "tsx examples/ops-pipeline/index.ts", + "pipeline:demo:trigger": "tsx examples/ops-pipeline/mocks/sendSyntheticError.ts", "prepublishOnly": "npm run build", "docs:build": "typedoc", "docs:verify-snippets": "tsx scripts/verify-docs-snippets.ts" diff --git a/src/spec/generators/__fixtures__/exampleSpec.ts b/src/spec/generators/__fixtures__/exampleSpec.ts new file mode 100644 index 000000000..1c88e8013 --- /dev/null +++ b/src/spec/generators/__fixtures__/exampleSpec.ts @@ -0,0 +1,28 @@ +/** + * Shared fixture AgentSpec reused by every cross-harness generator's tests + * (LOU-J1/J2/J3), so the generators are all verified against the exact same + * input. + */ +import { AgentSpec } from '../../schema'; + +export const exampleAgentSpec: AgentSpec = { + name: 'ops-fixer', + prompt: + 'You are an on-call fixer agent. Diagnose the root cause from the provided logs and ' + + 'produce a minimal unified diff that fixes it. Never invent files that were not shown to you.', + provider: { + type: 'mock', + model: 'mock-model-1', + }, + tools: ['http', 'current-date', 'day-name'], + policy: { + requiresApproval: true, + guardrails: ['secret-scan', 'diff-size-cap'], + }, + triggers: [ + { + type: 'webhook', + path: '/webhook', + }, + ], +}; diff --git a/src/spec/generators/claude-code.test.ts b/src/spec/generators/claude-code.test.ts new file mode 100644 index 000000000..5bece4755 --- /dev/null +++ b/src/spec/generators/claude-code.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest'; +import { generateClaudeCodeSkill } from './claude-code'; +import { exampleAgentSpec } from './__fixtures__/exampleSpec'; + +describe('generateClaudeCodeSkill', () => { + it('produces a SKILL.md path under .claude/skills/', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.path).toBe('.claude/skills/ops-fixer/SKILL.md'); + }); + + it('includes valid YAML frontmatter with name and description', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.content.startsWith('---\n')).toBe(true); + expect(file.content).toMatch(/name: "ops-fixer"/); + expect(file.content).toMatch(/description: "/); + }); + + it('contains the agent name', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.content).toContain(exampleAgentSpec.name); + }); + + it('contains the FULL prompt verbatim', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + expect(file.content).toContain(exampleAgentSpec.prompt); + }); + + it('contains every tool', () => { + const file = generateClaudeCodeSkill(exampleAgentSpec); + for (const t of exampleAgentSpec.tools ?? []) { + expect(file.content).toContain(t); + } + }); + + it('handles a spec with no tools', () => { + const file = generateClaudeCodeSkill({ ...exampleAgentSpec, tools: undefined }); + expect(file.content).toContain('(none)'); + }); +}); diff --git a/src/spec/generators/claude-code.ts b/src/spec/generators/claude-code.ts new file mode 100644 index 000000000..6aec58bf6 --- /dev/null +++ b/src/spec/generators/claude-code.ts @@ -0,0 +1,115 @@ +/** + * Claude Code generator (LOU-J1) + * + * Turns a cross-harness AgentSpec (LOU-H9, extended with `policy`/`triggers` + * in LOU-J1) into a real Claude Code Skill file (SKILL.md). + * + * This repository does not itself ship a `.claude/skills/` directory (there + * is nothing under `.claude/` to mirror the frontmatter shape from), so the + * frontmatter below follows Claude Code's own documented Skill format: a + * YAML frontmatter block with the two fields every skill needs - `name` + * (a short identifier) and `description` (what the skill does / when to use + * it, since that's what Claude Code's skill picker matches against) - + * followed by the skill's instructions as Markdown body content. + */ +import { AgentSpec } from '../schema'; + +/** + * A generated file: where it should be written, and its full content. + */ +export interface GeneratedFile { + path: string; + content: string; +} + +/** + * Slugifies an AgentSpec name into a filesystem/frontmatter-safe skill + * name: lowercase, spaces/underscores collapsed to single hyphens, and + * anything that isn't a-z/0-9/hyphen stripped. + */ +function slugify(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Builds a one-line description for the SKILL.md frontmatter from the + * spec's prompt: the first sentence (or the whole prompt if it has none), + * truncated to a reasonable frontmatter-friendly length. + */ +function buildDescription(prompt: string): string { + const firstSentence = prompt.split(/(?<=[.!?])\s/)[0] || prompt; + const trimmed = firstSentence.trim(); + return trimmed.length > 200 ? `${trimmed.slice(0, 197)}...` : trimmed; +} + +/** + * Escapes a value for safe embedding in a single YAML frontmatter scalar + * (wraps in double quotes, escapes embedded double quotes/backslashes). + */ +function yamlScalar(value: string): string { + return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`; +} + +/** + * Generates a Claude Code SKILL.md for `spec`. + * + * The full spec prompt is preserved verbatim in the Markdown body (not + * truncated) - only the frontmatter `description` is shortened, since that + * field exists purely for the skill picker, not to carry the whole prompt. + * Every tool from `spec.tools` is listed explicitly so a reader (or the + * LOU-J1 test) can see exactly which tools this skill expects to be + * available. + */ +export function generateClaudeCodeSkill(spec: AgentSpec): GeneratedFile { + const slug = slugify(spec.name); + const description = buildDescription(spec.prompt); + const tools = spec.tools ?? []; + + const frontmatter = [ + '---', + `name: ${yamlScalar(spec.name)}`, + `description: ${yamlScalar(description)}`, + '---', + ].join('\n'); + + const toolsSection = + tools.length > 0 + ? ['## Tools', '', ...tools.map((t) => `- \`${t}\``)].join('\n') + : ['## Tools', '', '(none)'].join('\n'); + + const policySection = spec.policy + ? [ + '## Policy', + '', + spec.policy.requiresApproval + ? '- Tool calls from this agent require human approval before running.' + : '- Tool calls from this agent do not require approval.', + ...(spec.policy.guardrails && spec.policy.guardrails.length > 0 + ? [`- Guardrails: ${spec.policy.guardrails.join(', ')}`] + : []), + ].join('\n') + : ''; + + const body = [ + frontmatter, + '', + `# ${spec.name}`, + '', + '## Prompt', + '', + spec.prompt, + '', + toolsSection, + ...(policySection ? ['', policySection] : []), + '', + ].join('\n'); + + return { + path: `.claude/skills/${slug}/SKILL.md`, + content: body, + }; +} diff --git a/src/spec/generators/codex.test.ts b/src/spec/generators/codex.test.ts new file mode 100644 index 000000000..89ef365ed --- /dev/null +++ b/src/spec/generators/codex.test.ts @@ -0,0 +1,42 @@ +import { describe, it, expect } from 'vitest'; +import { generateCodexConfig, CodexAgentConfig } from './codex'; +import { exampleAgentSpec } from './__fixtures__/exampleSpec'; + +describe('generateCodexConfig', () => { + it('produces a config path under .codex/agents/', () => { + const file = generateCodexConfig(exampleAgentSpec); + expect(file.path).toBe('.codex/agents/ops-fixer.json'); + }); + + it('produces valid, parseable JSON', () => { + const file = generateCodexConfig(exampleAgentSpec); + expect(() => JSON.parse(file.content)).not.toThrow(); + }); + + it("parsed output's fields match the fixture", () => { + const file = generateCodexConfig(exampleAgentSpec); + const parsed = JSON.parse(file.content) as CodexAgentConfig; + + expect(parsed.name).toBe(exampleAgentSpec.name); + expect(parsed.instructions).toBe(exampleAgentSpec.prompt); + expect(parsed.model).toEqual({ + provider: exampleAgentSpec.provider.type, + name: exampleAgentSpec.provider.model, + }); + expect(parsed.tools).toEqual(exampleAgentSpec.tools); + expect(parsed.policy).toEqual(exampleAgentSpec.policy); + expect(parsed.triggers).toEqual(exampleAgentSpec.triggers); + }); + + it('omits tools/policy/triggers gracefully when absent from the spec', () => { + const file = generateCodexConfig({ + name: 'bare', + prompt: 'bare prompt', + provider: { type: 'mock', model: 'mock-model-1' }, + }); + const parsed = JSON.parse(file.content) as CodexAgentConfig; + expect(parsed.tools).toEqual([]); + expect(parsed.policy).toBeUndefined(); + expect(parsed.triggers).toBeUndefined(); + }); +}); diff --git a/src/spec/generators/codex.ts b/src/spec/generators/codex.ts new file mode 100644 index 000000000..6544cfe41 --- /dev/null +++ b/src/spec/generators/codex.ts @@ -0,0 +1,67 @@ +/** + * Codex generator (LOU-J2) + * + * Turns an AgentSpec into a Codex-style agent-config JSON file. + * + * Uncertainty note: OpenAI Codex CLI's agent-config format is not something + * this repo has a copy of or a dependency on to verify against precisely, + * and no network research was performed as part of writing this generator. + * The shape below follows the LOU-J2 ticket's own illustrative shape as the + * best-effort baseline: a top-level `instructions` field carries the + * prompt (matching Codex's documented convention of an `instructions` + * field for agent system prompts, as opposed to Claude Code's `prompt`/ + * skill-body convention), `model`/`provider` carry the target model, + * `tools` is a flat array of tool names, and `policy`/`triggers` are + * passed through verbatim when present. If Codex's real schema differs + * (e.g. different field names or nesting), this generator's output shape + * will need to be adjusted to match - flagged here rather than silently + * assumed correct. + */ +import { AgentSpec } from '../schema'; +import { GeneratedFile } from './claude-code'; + +/** + * The Codex agent-config JSON shape this generator produces. See the + * uncertainty note above this generator's doc comment. + */ +export interface CodexAgentConfig { + name: string; + instructions: string; + model: { + provider: string; + name: string; + }; + tools: string[]; + policy?: AgentSpec['policy']; + triggers?: AgentSpec['triggers']; +} + +function slugify(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Generates a Codex agent-config JSON file for `spec`. + */ +export function generateCodexConfig(spec: AgentSpec): GeneratedFile { + const config: CodexAgentConfig = { + name: spec.name, + instructions: spec.prompt, + model: { + provider: spec.provider.type, + name: spec.provider.model, + }, + tools: spec.tools ?? [], + ...(spec.policy ? { policy: spec.policy } : {}), + ...(spec.triggers ? { triggers: spec.triggers } : {}), + }; + + return { + path: `.codex/agents/${slugify(spec.name)}.json`, + content: JSON.stringify(config, null, 2) + '\n', + }; +} diff --git a/src/spec/generators/pi.test.ts b/src/spec/generators/pi.test.ts new file mode 100644 index 000000000..51fd66127 --- /dev/null +++ b/src/spec/generators/pi.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect } from 'vitest'; +import { parse as parseYaml } from 'yaml'; +import { generatePiSkill, PiSkillDocument } from './pi'; +import { exampleAgentSpec } from './__fixtures__/exampleSpec'; + +describe('generatePiSkill', () => { + it('produces a YAML file path under .pi/skills/', () => { + const file = generatePiSkill(exampleAgentSpec); + expect(file.path).toBe('.pi/skills/ops-fixer.yaml'); + }); + + it('produces valid, parseable YAML', () => { + const file = generatePiSkill(exampleAgentSpec); + expect(() => parseYaml(file.content)).not.toThrow(); + }); + + it('contains name, full prompt, and every tool', () => { + const file = generatePiSkill(exampleAgentSpec); + const parsed = parseYaml(file.content) as PiSkillDocument; + + expect(parsed.name).toBe(exampleAgentSpec.name); + expect(parsed.prompt).toBe(exampleAgentSpec.prompt); + expect(file.content).toContain(exampleAgentSpec.name); + expect(file.content).toContain(exampleAgentSpec.prompt.split('\n')[0].slice(0, 20)); + for (const t of exampleAgentSpec.tools ?? []) { + expect(parsed.tools).toContain(t); + } + }); + + it('round-trips provider/policy/triggers', () => { + const file = generatePiSkill(exampleAgentSpec); + const parsed = parseYaml(file.content) as PiSkillDocument; + expect(parsed.provider).toEqual(exampleAgentSpec.provider); + expect(parsed.policy).toEqual(exampleAgentSpec.policy); + expect(parsed.triggers).toEqual(exampleAgentSpec.triggers); + }); +}); diff --git a/src/spec/generators/pi.ts b/src/spec/generators/pi.ts new file mode 100644 index 000000000..881c95a5a --- /dev/null +++ b/src/spec/generators/pi.ts @@ -0,0 +1,53 @@ +/** + * Pi generator (LOU-J3) + * + * Turns an AgentSpec into a Pi skill YAML file. Reuses the `yaml` package + * that's already a project dependency (added for loadSpec.ts's .yaml/.yml + * parsing - see src/spec/loadSpec.ts) for stringifying, rather than adding + * a new YAML dependency just for this generator. + */ +import { stringify as stringifyYaml } from 'yaml'; +import { AgentSpec } from '../schema'; +import { GeneratedFile } from './claude-code'; + +function slugify(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * The plain-data shape serialized to YAML for a Pi skill. Kept close to + * AgentSpec itself (Pi's skill format, like the source AgentSpec, is a + * name/prompt/tools/provider/policy/triggers record) rather than + * introducing an unrelated schema. + */ +export interface PiSkillDocument { + name: string; + prompt: string; + tools: string[]; + provider: AgentSpec['provider']; + policy?: AgentSpec['policy']; + triggers?: AgentSpec['triggers']; +} + +/** + * Generates a Pi skill YAML file for `spec`. + */ +export function generatePiSkill(spec: AgentSpec): GeneratedFile { + const doc: PiSkillDocument = { + name: spec.name, + prompt: spec.prompt, + tools: spec.tools ?? [], + provider: spec.provider, + ...(spec.policy ? { policy: spec.policy } : {}), + ...(spec.triggers ? { triggers: spec.triggers } : {}), + }; + + return { + path: `.pi/skills/${slugify(spec.name)}.yaml`, + content: stringifyYaml(doc), + }; +} diff --git a/src/spec/schema.ts b/src/spec/schema.ts index 9ed2c489a..342e69265 100644 --- a/src/spec/schema.ts +++ b/src/spec/schema.ts @@ -14,11 +14,46 @@ export interface AgentSpecProvider { model: string; } +/** + * Optional cross-harness execution policy (LOU-J1+). Deliberately loose - + * different target harnesses (Claude Code, Codex, Pi, ...) each honor a + * different subset of this, and new fields are expected to be added by + * later generators without needing a schema migration every time, so this + * is intentionally an open record (`.passthrough()` on the zod side) rather + * than a closed, harness-specific shape. + */ +export interface AgentSpecPolicy { + /** Whether tool calls from this agent require human approval before running (LOU-C). */ + requiresApproval?: boolean; + /** Named guardrails (see src/execution/guardrails.ts) this agent's actions must pass. */ + guardrails?: string[]; + /** Additional, harness-specific policy fields. */ + [key: string]: unknown; +} + +/** + * Optional trigger describing when/how this agent is invoked outside of a + * direct call (e.g. a monitoring webhook, a cron schedule). Open record for + * the same reason as AgentSpecPolicy above. + */ +export interface AgentSpecTrigger { + type: string; + [key: string]: unknown; +} + export interface AgentSpec { name: string; prompt: string; provider: AgentSpecProvider; tools?: string[]; + /** + * Optional (LOU-J1+, backward-compatible with LOU-H9's original + * {name, prompt, provider, tools} shape - existing specs that omit this + * still validate and load unchanged). + */ + policy?: AgentSpecPolicy; + /** Optional (LOU-J1+), same backward-compatibility note as `policy`. */ + triggers?: AgentSpecTrigger[]; } export const agentSpecProviderSchema = z.object({ @@ -30,6 +65,21 @@ export const agentSpecProviderSchema = z.object({ }), }); +export const agentSpecPolicySchema = z + .object({ + requiresApproval: z.boolean().optional(), + guardrails: z.array(z.string()).optional(), + }) + .passthrough(); + +export const agentSpecTriggerSchema = z + .object({ + type: z.string({ + required_error: "AgentSpec validation failed: missing required field 'triggers[].type'", + }), + }) + .passthrough(); + export const agentSpecSchema = z.object({ name: z.string({ required_error: "AgentSpec validation failed: missing required field 'name'", @@ -39,4 +89,6 @@ export const agentSpecSchema = z.object({ }), provider: agentSpecProviderSchema, tools: z.array(z.string()).optional(), + policy: agentSpecPolicySchema.optional(), + triggers: z.array(agentSpecTriggerSchema).optional(), }); diff --git a/src/tools/built-in/slack.test.ts b/src/tools/built-in/slack.test.ts new file mode 100644 index 000000000..05800fbc7 --- /dev/null +++ b/src/tools/built-in/slack.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, vi } from 'vitest'; +import { buildSlackAlertPayload, createSlackTool, postSlackAlert } from './slack'; + +function mockFetchOk() { + return vi.fn().mockResolvedValue({ + ok: true, + status: 200, + text: async () => '', + } as Response); +} + +describe('buildSlackAlertPayload', () => { + it('produces a payload with exactly one interactive button carrying the correct approvalId', () => { + const payload = buildSlackAlertPayload('#incidents', 'Deploy failed', 'approval-123'); + + const actionsBlocks = payload.blocks.filter((b) => b.type === 'actions'); + expect(actionsBlocks).toHaveLength(1); + + const buttons = actionsBlocks[0].elements ?? []; + expect(buttons).toHaveLength(1); + expect(buttons[0].action_id).toBe('fix_it'); + expect(buttons[0].value).toBe('approval-123'); + expect(buttons[0].text?.text).toBe('Fix it'); + }); + + it('includes the channel and message text', () => { + const payload = buildSlackAlertPayload('#incidents', 'Deploy failed', 'approval-123'); + expect(payload.channel).toBe('#incidents'); + expect(payload.text).toBe('Deploy failed'); + }); +}); + +describe('postSlackAlert / createSlackTool (mocked Slack API)', () => { + it('posts the built payload to the configured webhook URL', async () => { + const fetchImpl = mockFetchOk(); + + await postSlackAlert('#incidents', 'Deploy failed', 'approval-123', { + webhookUrl: 'https://hooks.slack.test/services/mock', + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe('https://hooks.slack.test/services/mock'); + const body = JSON.parse((init as RequestInit).body as string); + expect(body.blocks[1].elements[0].value).toBe('approval-123'); + }); + + it('throws a clear error when no webhook URL is configured', async () => { + const original = process.env.SLACK_WEBHOOK_URL; + delete process.env.SLACK_WEBHOOK_URL; + try { + await expect( + postSlackAlert('#incidents', 'msg', 'approval-1', { fetchImpl: mockFetchOk() as unknown as typeof fetch }) + ).rejects.toThrow(/SLACK_WEBHOOK_URL/); + } finally { + if (original !== undefined) process.env.SLACK_WEBHOOK_URL = original; + } + }); + + it('the ToolDescriptor wraps an AI SDK tool() with the expected required params', async () => { + const fetchImpl = mockFetchOk(); + const descriptor = createSlackTool({ + webhookUrl: 'https://hooks.slack.test/services/mock', + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(descriptor.tool).toBeDefined(); + const result = await descriptor.tool.execute!( + { channel: '#incidents', message: 'Deploy failed', approvalId: 'approval-123' }, + {} as any + ); + + expect(result).toEqual({ ok: true }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/tools/built-in/slack.ts b/src/tools/built-in/slack.ts new file mode 100644 index 000000000..0c99fcf6f --- /dev/null +++ b/src/tools/built-in/slack.ts @@ -0,0 +1,155 @@ +/** + * Slack alert tool (LOU-J5) + * + * Posts an alert message to a Slack channel via an Incoming Webhook, using + * Slack's Block Kit format with a single interactive "Fix it" button whose + * `value` carries an approvalId. Clicking that button is meant to resolve a + * REAL LOU-C ApprovalGate PendingApproval (see resumeAfterApproval() in + * src/execution/resume.ts, and examples/ops-pipeline/slackInteractions.ts + * for the interaction-callback route that wires the button click back to + * it) - this tool itself only sends the message; it does not bypass or + * reimplement the approval gate. + * + * Follows the same ToolDescriptor pattern established in + * src/tools/built-in/http.ts and github.ts: wraps an AI SDK `tool()` call + * (not a plain object literal). + * + * ## Env var convention (LOU-F8 style) + * + * The webhook URL is read from `SLACK_WEBHOOK_URL` by default - following + * the same "credential lives in an env var named after the provider" + * convention resolveProvider.ts's PROVIDER_ENV_TABLE uses for LLM + * providers (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`). + */ +import { tool } from 'ai'; +import { z } from 'zod'; +import { ToolDescriptor } from '../../types'; + +export const SLACK_WEBHOOK_URL_ENV_KEY = 'SLACK_WEBHOOK_URL'; + +/** + * A single Slack Block Kit block - kept intentionally minimal (only the + * shapes this tool actually emits: section and actions blocks with a + * button element) rather than modeling the whole Block Kit schema. + */ +export interface SlackBlock { + type: string; + text?: { type: string; text: string }; + elements?: Array<{ + type: string; + text?: { type: string; text: string }; + action_id?: string; + value?: string; + style?: string; + }>; +} + +export interface SlackAlertPayload { + channel: string; + text: string; + blocks: SlackBlock[]; +} + +/** + * Builds the Slack Block Kit payload for an alert with a "Fix it" button. + * Exported so callers (and tests) can inspect the exact payload shape + * without going through a real/mocked HTTP call. + */ +export function buildSlackAlertPayload( + channel: string, + message: string, + approvalId: string +): SlackAlertPayload { + return { + channel, + text: message, + blocks: [ + { + type: 'section', + text: { type: 'mrkdwn', text: message }, + }, + { + type: 'actions', + elements: [ + { + type: 'button', + text: { type: 'plain_text', text: 'Fix it' }, + action_id: 'fix_it', + value: approvalId, + style: 'primary', + }, + ], + }, + ], + }; +} + +export interface SlackToolOptions { + /** Overrides SLACK_WEBHOOK_URL for testing/injection. */ + webhookUrl?: string; + /** Overrides the fetch implementation used to POST the payload (testing). */ + fetchImpl?: typeof fetch; +} + +/** + * Posts a pre-built Slack alert payload to the configured Incoming Webhook + * URL. Exported standalone (in addition to the ToolDescriptor below) so + * examples/ops-pipeline code that isn't going through AgentExecutor's tool + * plumbing can still send an alert directly. + */ +export async function postSlackAlert( + channel: string, + message: string, + approvalId: string, + options: SlackToolOptions = {} +): Promise<{ ok: boolean }> { + const webhookUrl = options.webhookUrl ?? process.env[SLACK_WEBHOOK_URL_ENV_KEY]; + if (!webhookUrl) { + throw new Error( + `Slack tool: no webhook URL configured. Set ${SLACK_WEBHOOK_URL_ENV_KEY} or pass options.webhookUrl.` + ); + } + + const doFetch = options.fetchImpl ?? fetch; + const payload = buildSlackAlertPayload(channel, message, approvalId); + + const response = await doFetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Slack tool: webhook post failed: ${response.statusText} - ${errorText}`); + } + + return { ok: true }; +} + +/** + * Creates the Slack alert ToolDescriptor. Requires {channel, message, + * approvalId} - `approvalId` is the id of a REAL LOU-C PendingApproval + * (see ApprovalGate.ts) that the "Fix it" button, once clicked, resolves + * via resumeAfterApproval(). + */ +export function createSlackTool(options: SlackToolOptions = {}): ToolDescriptor { + return { + displayName: 'Post Slack Alert', + tool: tool({ + description: + 'Posts an alert to a Slack channel with a "Fix it" button that resumes a pending approval.', + parameters: z.object({ + channel: z.string().describe('Slack channel to post to (e.g. "#incidents")'), + message: z.string().describe('Alert message text'), + approvalId: z.string().describe('The pending approval id the "Fix it" button will resolve'), + }), + execute: async ({ channel, message, approvalId }) => { + return postSlackAlert(channel, message, approvalId, options); + }, + }), + }; +} + +/** Default Slack tool instance, reading SLACK_WEBHOOK_URL at call time. */ +export const slackTool: ToolDescriptor = createSlackTool();