feat: a refuted-then-verified goal now teaches the next one

docs/harness-audit.md

@@ -38,9 +38,10 @@ If the failure mode has been absorbed by model capability, the component is now
 | 14 | Planner → executor model swap mid-turn | `loop.ts` lines 773–784 | "Cheap executor model can follow a plan made by an expensive planner" | Capability | **Coupled to #1** — if #1 goes, this goes too | Covered by #1 test |
 | 15 | Brain auto-recall | `loop.ts` lines 537–563 | "Model can't always remember entities from earlier in the same session" | Capability | **Low load** — nice UX, probably not critical. Test by disabling and measuring continuity. | Needs new env flag |
 | 16 | Weak-model tool inventory injection | `loop.ts` lines 800–808 | "Weak / free models invent tool names when not given an explicit list" | Capability | **Holds for free tier**, likely **stale for Sonnet / Opus** | Split injection on model tier |
+| 18 | Goal-outcome lessons (experience library) | `src/learnings/experience.ts` | "A single verified run is not evidence; a refuted→verified verifier contrast on the same objective is. Models repeat the mistake that cost them a round unless the fix is carried into the next goal's prompt." (from Harness², arXiv 2610.14224) | Capability | **Just added** — needs burn-in; lessons are scoped per wallet/workspace and retire when later goals refute while they were in play. | `FRANKLIN_NO_EXPERIENCE=1` (wired; `no-experience` bench config) |
 | 17 | Sub-agent delegation (Task tool) | `src/tools/subagent.ts` | "Main agent's context can't afford broad research; delegate to a fresh sub-agent" | Capability / context mgmt | **Holds** — basic parallelism + context isolation win | Not a candidate |
 
-**Count:** 17 components.
+**Count:** 18 components.
 - **Permanent / infrastructure** (not candidates for removal): 10 (#6–13, #17)
 - **Capability-gap hedges worth re-testing now**: 7 (#1, #2, #4, #5, #14, #15, #16)
 - **Just-added, needs burn-in**: 1 (#4)
@@ -113,3 +114,4 @@ The correct order is: **observe → map to assumption → check if existing comp
 ## Changelog
 
 - **2026-04-21** — Initial audit on v3.8.14. 17 components enumerated. 5 ablation candidates ranked. Bench script scaffolded.
+- **2026-10-18** — #18 goal-outcome lessons added (Harness²-style harnessing-level recursion). `no-experience` bench config added.

package.json

@@ -38,7 +38,7 @@
     "desktop:package:win": "npm run dist:win --workspace @blockrun/franklin-desktop",
     "dev": "tsc --watch",
     "start": "node dist/index.js",
-    "test": "npm run build && node --test --test-concurrency=4 --test-reporter=spec test/local.mjs test/api-key.local.mjs test/solana-migration.local.mjs test/serve-security.local.mjs test/skills.local.mjs test/repair.mjs test/model-resolver.mjs test/exa.local.mjs test/audit-batch.local.mjs test/polymarket.local.mjs test/market.local.mjs test/hooks.local.mjs test/scheduler.local.mjs test/trade-plan.local.mjs test/goal.local.mjs test/memory.local.mjs test/agent-host.local.mjs test/reservation.local.mjs test/model-catalog.local.mjs test/arc.local.mjs test/subprocess-env.local.mjs test/withdraw.local.mjs test/orders.local.mjs test/payment-hold.local.mjs",
+    "test": "npm run build && node --test --test-concurrency=4 --test-reporter=spec test/local.mjs test/api-key.local.mjs test/solana-migration.local.mjs test/serve-security.local.mjs test/skills.local.mjs test/repair.mjs test/model-resolver.mjs test/exa.local.mjs test/audit-batch.local.mjs test/polymarket.local.mjs test/market.local.mjs test/hooks.local.mjs test/scheduler.local.mjs test/trade-plan.local.mjs test/goal.local.mjs test/memory.local.mjs test/agent-host.local.mjs test/reservation.local.mjs test/model-catalog.local.mjs test/arc.local.mjs test/subprocess-env.local.mjs test/withdraw.local.mjs test/orders.local.mjs test/payment-hold.local.mjs test/experience.local.mjs",
     "test:e2e": "npm run build && node --test --test-reporter=spec test/e2e.mjs",
     "e2e:polymarket:readonly": "npm run build && node scripts/polymarket-e2e-readonly.mjs",
     "test:free-models": "npm run build && node --test --test-reporter=spec test/free-model-matrix.mjs",

scripts/harness-bench.mjs

@@ -74,6 +74,7 @@ const CONFIGS = [
   { name: 'no-plan', env: { FRANKLIN_NOPLAN: '1' }, summary: 'plan-then-execute OFF' },
   { name: 'no-dynamic-tools', env: { FRANKLIN_DYNAMIC_TOOLS: '0' }, summary: 'ActivateTool / CORE gating OFF' },
   { name: 'no-eval', env: { FRANKLIN_NO_EVAL: '1' }, summary: 'Groundedness evaluator OFF' },
+  { name: 'no-experience', env: { FRANKLIN_NO_EXPERIENCE: '1' }, summary: 'Goal-outcome lessons OFF' },
 ];
 
 // ─── Per-run settings ────────────────────────────────────────────────────

src/agent/commands.ts

@@ -995,11 +995,28 @@ export async function handleSlashCommand(
   // /learnings — view or clear per-user learnings
   if (input === '/learnings' || input.startsWith('/learnings ')) {
     const { loadLearnings, decayLearnings, saveLearnings } = await import('../learnings/store.js');
+    const { loadExperiences, saveExperiences } = await import('../learnings/experience.js');
     const arg = input.slice('/learnings'.length).trim();
-    if (arg === 'clear') {
+    if (arg === 'clear lessons') {
+      saveExperiences([]);
+      ctx.onEvent({ kind: 'text_delta', text: 'All verified lessons cleared.\n' });
+    } else if (arg === 'clear') {
       saveLearnings([]);
-      ctx.onEvent({ kind: 'text_delta', text: 'All learnings cleared.\n' });
+      ctx.onEvent({ kind: 'text_delta', text: 'All learnings cleared (verified lessons kept — `/learnings clear lessons` removes those).\n' });
     } else {
+      const lessons = loadExperiences();
+      if (lessons.length > 0) {
+        const live = lessons.filter(l => l.verdict !== 'confounded');
+        const confounded = lessons.length - live.length;
+        let text = `**Lessons verified by goal outcomes** (${live.length}${confounded ? `, ${confounded} confounded` : ''})\n\n`;
+        for (const l of live.sort((a, b) => b.times_confirmed - a.times_confirmed)) {
+          const where = l.scope.wallet ? `wallet ${l.scope.wallet}` : `workspace ${l.scope.workspace}`;
+          text += `  [${l.verdict}/${l.confidence}] ${l.lesson}\n`;
+          text += `      when: ${l.cue}\n`;
+          text += `      evidence: goal ${l.evidence.goalId}, ${l.evidence.roundsRefuted} refuted round(s), closed: ${l.evidence.closedGaps.join('; ') || '—'} · ${where} · ×${l.times_confirmed}\n`;
+        }
+        ctx.onEvent({ kind: 'text_delta', text: text + '\n' });
+      }
       let learnings = loadLearnings();
       if (learnings.length === 0) {
         ctx.onEvent({ kind: 'text_delta', text: 'No learnings yet. Franklin learns your preferences over time.\n' });

src/agent/context.ts

@@ -11,6 +11,8 @@ import { getWalletAddress as getBaseWalletAddress } from '@blockrun/llm';
 import { Keypair } from '@solana/web3.js';
 import bs58 from 'bs58';
 import { loadLearnings, decayLearnings, saveLearnings, formatForPrompt } from '../learnings/store.js';
+import { renderExperienceSection, experienceScope } from '../learnings/experience.js';
+import { scanForInjection } from './injection-scan.js';
 import { isKeyMode, resolvePayMode } from '../payments/auth-mode.js';
 import { sanitizeSubprocessEnv } from '../tools/subprocess-env.js';
 
@@ -584,6 +586,27 @@ export function assembleInstructions(workingDir: string, model?: string): string
     }
   } catch { /* learnings are optional — never block startup */ }
 
+  // Outcome-verified lessons (src/learnings/experience.ts): written only when
+  // a /goal was refuted and then verified, scoped to this wallet or workspace.
+  // Rendered after the learnings block and inside the same personal-context
+  // budget until the always-on cap lands. FRANKLIN_NO_EXPERIENCE=1 removes
+  // the section (ADR-0003).
+  try {
+    const walletInfo = readRuntimeWallet();
+    const scopes = [
+      experienceScope(workingDir, walletInfo, 'general'),
+      experienceScope(workingDir, walletInfo, 'trading'),
+    ];
+    const seen = new Set<string>();
+    for (const scope of scopes) {
+      const key = scope.wallet ? `w:${scope.wallet}` : `d:${scope.workspace}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      const section = renderExperienceSection(scope);
+      if (section) parts.push(section);
+    }
+  } catch { /* experience is optional — never block startup */ }
+
   // Procedural skills (bundled + learned + user + project) used to be
   // injected here at session boot. They now flow through the unified
   // src/skills/ Registry and are matched per-turn against the user's
@@ -677,59 +700,6 @@ export function invalidateInstructionCache(workingDir?: string): void {
   }
 }
 
-// ─── Prompt Injection Detection ────────────────────────────────────────────
-
-/** Patterns that indicate potential prompt injection in context files. */
-const INJECTION_PATTERNS: Array<{ pattern: RegExp; description: string }> = [
-  // Direct instruction override attempts
-  { pattern: /ignore\s+(all\s+)?previous\s+instructions/i, description: 'instruction override' },
-  { pattern: /disregard\s+(all\s+)?(previous\s+|above\s+)?rules/i, description: 'rule disregard' },
-  { pattern: /forget\s+(everything|all|your)\s+(you|instructions|rules)/i, description: 'memory wipe' },
-  { pattern: /you\s+are\s+now\s+(?:a\s+)?(?:different|new|unrestricted)/i, description: 'identity hijack' },
-  { pattern: /system\s*:\s*you\s+are/i, description: 'fake system message' },
-  // Dangerous command injection
-  { pattern: /execute\s+(curl|wget|bash|sh|python|node)\b/i, description: 'command execution' },
-  { pattern: /\bcat\s+\/etc\/(passwd|shadow|sudoers)/i, description: 'credential access' },
-  { pattern: /\brm\s+-rf\s+[\/~]/i, description: 'destructive command' },
-  { pattern: /\beval\s*\(/i, description: 'eval injection' },
-  // Data exfiltration
-  { pattern: /\bcurl\s+.*\|\s*(bash|sh)/i, description: 'pipe to shell' },
-  { pattern: /send\s+(to|via)\s+(http|webhook|url)/i, description: 'data exfiltration' },
-  // HTML/comment injection
-  { pattern: /<!--[\s\S]*?-->/g, description: 'HTML comment injection' },
-];
-
-/** Invisible unicode characters that can hide malicious content. */
-const INVISIBLE_UNICODE = /[\u200B-\u200F\u202A-\u202E\u2060-\u2064\uFEFF\u00AD]/g;
-
-/**
- * Scan text for prompt injection patterns and invisible unicode.
- * Returns sanitized text with threats neutralized and a list of detections.
- */
-function scanForInjection(text: string): { sanitized: string; threats: string[] } {
-  const threats: string[] = [];
-  let sanitized = text;
-
-  // Check for invisible unicode
-  if (INVISIBLE_UNICODE.test(sanitized)) {
-    const count = (sanitized.match(INVISIBLE_UNICODE) || []).length;
-    threats.push(`${count} invisible unicode character(s) removed`);
-    sanitized = sanitized.replace(INVISIBLE_UNICODE, '');
-  }
-
-  // Check for injection patterns
-  for (const { pattern, description } of INJECTION_PATTERNS) {
-    const matches = sanitized.match(pattern);
-    if (matches) {
-      threats.push(`${description}: "${matches[0].slice(0, 50)}"`);
-      // Neutralize by wrapping in brackets (visible but defanged)
-      sanitized = sanitized.replace(pattern, (match) => `[BLOCKED: ${match}]`);
-    }
-  }
-
-  return { sanitized, threats };
-}
-
 // ─── Project Config ────────────────────────────────────────────────────────
 
 /**
@@ -801,7 +771,7 @@ function buildEnvironmentSection(workingDir: string): string {
   return lines.join('\n');
 }
 
-function readRuntimeWallet(): { chain?: string; base?: string; solana?: string } {
+export function readRuntimeWallet(): { chain?: string; base?: string; solana?: string } {
   const home = process.env.HOME || '';
   if (!home) return {};
   const blockrunDir = path.join(home, '.blockrun');

src/agent/injection-scan.ts

@@ -0,0 +1,59 @@
+/**
+ * Prompt-injection scanning for text that enters the system prompt from
+ * outside Franklin's own code: project config files, and lessons written
+ * back by the goal-outcome reflector. Shared so every such path applies the
+ * same patterns.
+ */
+
+// ─── Prompt Injection Detection ────────────────────────────────────────────
+
+/** Patterns that indicate potential prompt injection in context files. */
+const INJECTION_PATTERNS: Array<{ pattern: RegExp; description: string }> = [
+  // Direct instruction override attempts
+  { pattern: /ignore\s+(all\s+)?previous\s+instructions/i, description: 'instruction override' },
+  { pattern: /disregard\s+(all\s+)?(previous\s+|above\s+)?rules/i, description: 'rule disregard' },
+  { pattern: /forget\s+(everything|all|your)\s+(you|instructions|rules)/i, description: 'memory wipe' },
+  { pattern: /you\s+are\s+now\s+(?:a\s+)?(?:different|new|unrestricted)/i, description: 'identity hijack' },
+  { pattern: /system\s*:\s*you\s+are/i, description: 'fake system message' },
+  // Dangerous command injection
+  { pattern: /execute\s+(curl|wget|bash|sh|python|node)\b/i, description: 'command execution' },
+  { pattern: /\bcat\s+\/etc\/(passwd|shadow|sudoers)/i, description: 'credential access' },
+  { pattern: /\brm\s+-rf\s+[\/~]/i, description: 'destructive command' },
+  { pattern: /\beval\s*\(/i, description: 'eval injection' },
+  // Data exfiltration
+  { pattern: /\bcurl\s+.*\|\s*(bash|sh)/i, description: 'pipe to shell' },
+  { pattern: /send\s+(to|via)\s+(http|webhook|url)/i, description: 'data exfiltration' },
+  // HTML/comment injection
+  { pattern: /<!--[\s\S]*?-->/g, description: 'HTML comment injection' },
+];
+
+/** Invisible unicode characters that can hide malicious content. */
+const INVISIBLE_UNICODE = /[\u200B-\u200F\u202A-\u202E\u2060-\u2064\uFEFF\u00AD]/g;
+
+/**
+ * Scan text for prompt injection patterns and invisible unicode.
+ * Returns sanitized text with threats neutralized and a list of detections.
+ */
+export function scanForInjection(text: string): { sanitized: string; threats: string[] } {
+  const threats: string[] = [];
+  let sanitized = text;
+
+  // Check for invisible unicode
+  if (INVISIBLE_UNICODE.test(sanitized)) {
+    const count = (sanitized.match(INVISIBLE_UNICODE) || []).length;
+    threats.push(`${count} invisible unicode character(s) removed`);
+    sanitized = sanitized.replace(INVISIBLE_UNICODE, '');
+  }
+
+  // Check for injection patterns
+  for (const { pattern, description } of INJECTION_PATTERNS) {
+    const matches = sanitized.match(pattern);
+    if (matches) {
+      threats.push(`${description}: "${matches[0].slice(0, 50)}"`);
+      // Neutralize by wrapping in brackets (visible but defanged)
+      sanitized = sanitized.replace(pattern, (match) => `[BLOCKED: ${match}]`);
+    }
+  }
+
+  return { sanitized, threats };
+}

src/agent/loop.ts

@@ -731,6 +731,7 @@ async function runInteractiveSession(
     client,
     getModel: () => config.model,
     capabilities: config.capabilities,
+    getHistory: () => history,
   });
   if (config.resumeSessionId) {
     const resumedMeta = loadSessionMeta(config.resumeSessionId);
@@ -955,6 +956,13 @@ async function runInteractiveSession(
       const matches = matchSkillTriggers(input, skillRegistry.list());
       if (matches.length > 0) {
         turnSkillHints = formatSkillHints(matches);
+        // Remember which skills were in play for the active goal, so the
+        // goal-outcome reflector can tie a lesson to them later.
+        const hintedGoal = getActiveGoal();
+        if (hintedGoal && hintedGoal.status === 'active') {
+          const names = matches.map(m => m.skill.skill.name);
+          updateActiveGoal({ skillsHinted: [...new Set([...(hintedGoal.skillsHinted ?? []), ...names])].slice(0, 20) });
+        }
         if (config.debug && matches.length > 0) {
           const summary = matches.map(m => `${m.skill.skill.name}(${m.score.toFixed(1)})`).join(', ');
           onEvent({ kind: 'text_delta', text: `*[skill triggers] ${summary}*\n` });

src/goal/runtime.ts

@@ -6,7 +6,15 @@
  */
 
 import type { ModelClient } from '../agent/llm.js';
-import type { CapabilityHandler, CapabilityResult, ExecutionScope } from '../agent/types.js';
+import type { CapabilityHandler, CapabilityResult, Dialogue, ExecutionScope } from '../agent/types.js';
+import { readRuntimeWallet } from '../agent/context.js';
+import {
+  experienceEnabled,
+  experienceScope,
+  markRenderedLessonsRefuted,
+  recordConfoundedGoal,
+  reflectGoalExperience,
+} from '../learnings/experience.js';
 import {
   getActiveGoal,
   loadPlan,
@@ -20,12 +28,30 @@ import {
   runVerifierPanel,
   type GoalEngineContext,
 } from './engine.js';
-import { goalMaxRounds } from './types.js';
+import { goalMaxRounds, type GoalState } from './types.js';
 
 export interface GoalEngineDeps {
   client: ModelClient;
   getModel: () => string;
   capabilities: CapabilityHandler[];
+  /** The session transcript, read by the goal-outcome reflector as evidence. */
+  getHistory?: () => Dialogue[];
+}
+
+// The reflector runs after a verified claim without blocking the reply.
+// Tests (and shutdown) await it here.
+let pendingReflection: Promise<unknown> = Promise.resolve();
+
+export function awaitGoalReflection(): Promise<void> {
+  return pendingReflection.then(() => undefined, () => undefined);
+}
+
+function scopeFor(goal: { kind: GoalState['kind'] }, scope: ExecutionScope) {
+  return experienceScope(scope.workingDir, safeWallet(), goal.kind);
+}
+
+function safeWallet() {
+  try { return readRuntimeWallet(); } catch { return {}; }
 }
 
 let deps: GoalEngineDeps | null = null;
@@ -88,6 +114,7 @@ export async function processGoalClaim(
       status: 'blocked',
       blockedReason: `verification round cap (${goalMaxRounds()}) reached`,
     });
+    try { recordConfoundedGoal(goal, scopeFor(goal, scope)); } catch { /* best-effort */ }
     return {
       output:
         `Verification round cap reached (${goalMaxRounds()} rounds). The goal is now blocked — ` +
@@ -109,6 +136,19 @@ export async function processGoalClaim(
       summary = await runSummarizer(ctx, { ...goal, rounds }, planMd);
     } catch { /* summary is best-effort */ }
     updateActiveGoal({ status: 'completed', rounds, verifierGaps: [], summary });
+    // Harnessing-level recursion: a refuted→verified contrast is evidence; a
+    // round-1 pass is adoption alone and teaches nothing durable.
+    if (experienceEnabled() && (rounds.length >= 2 || goal.strategistNote)) {
+      const verified: GoalState = { ...goal, rounds, status: 'completed', summary };
+      pendingReflection = reflectGoalExperience({
+        goal: verified,
+        planMd,
+        history: deps?.getHistory?.() ?? [],
+        scope: scopeFor(goal, scope),
+        sessionId: scope.sessionId ?? '',
+        client: ctx.client,
+      }).catch(() => null);
+    }
     return {
       output:
         `GOAL VERIFIED COMPLETE (round ${round}, ${outcome.verdicts.length}-reviewer panel).\n\n` +
@@ -134,6 +174,9 @@ export async function processGoalClaim(
   }
 
   const updated = updateActiveGoal({ status: 'active', rounds, verifierGaps: outcome.gaps });
+  if (round === 1) {
+    try { markRenderedLessonsRefuted(scopeFor(goal, scope), goal.id); } catch { /* best-effort */ }
+  }
   let strategistLine = '';
   if (updated && needsStrategist(updated)) {
     try {

src/goal/types.ts

@@ -53,6 +53,10 @@ export interface GoalState {
   blockedReason?: string;
   /** Closing user-facing summary, once completed. */
   summary?: string;
+  /** Skills whose trigger hints fired on turns while this goal was active.
+   *  Carried into the experience record so a lesson can later be scored
+   *  against the skills that were in play. */
+  skillsHinted?: string[];
 }
 
 export interface VerifierVerdict {