From 252f84ca86711aa2909591f490c06e6f31f2db9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 09:57:29 +0100 Subject: [PATCH 01/12] fix(ts): a failure inside a referenced section lands in the document it was written in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A step spliced in by a reference block (ADR 0016) has spans in the oath it was WRITTEN in, but the run result named only the oath that ran it. The language server therefore placed the failure in the wrong file — against offsets that address some other sentence — or dropped it when the source hash didn't match. The run-result payload is version 2: - `failure.docPath` names the document `line`, `cells` and `anchor` are offsets into. Absent (the common case) means the oath itself. - `documents` carries a hash per referenced oath that contributed steps, so a consumer can tell a stale failure from a live one exactly as `sourceHash` does for the oath. - `lines` holds only the running oath's own lines: a spliced step's line is not in this file, and a line-wash renderer would decorate an unrelated sentence. The executor's stack frame names the same document, so an editor resolving a failure from the frame lands there too. The LSP publishes a spliced failure against the referenced document's URI — a shared oath has no result file of its own, so its failures previously had no way to reach the editor at all. conformance/run-results/expected.json grows a fourth example covering it, which is what the other six ports now have to satisfy. Ports-deferred: py, java, ruby, rust, dotnet, go — payload lands per port next Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- conformance/adapter/smoke.sh | 2 +- conformance/run-results/README.md | 12 +++- conformance/run-results/expected.json | 32 ++++++++++- typescript/packages/core/src/execute.ts | 10 +++- .../packages/core/src/failure-anchor.ts | 18 ++++++ typescript/packages/core/src/failure.ts | 10 +++- typescript/packages/core/src/result.ts | 15 ++++- .../packages/core/src/run-diagnostics.ts | 15 ++++- .../core/tests/failure-step-span.test.ts | 2 +- .../core/tests/run-diagnostics.test.ts | 39 ++++++++++++- .../packages/lsp/src/run-results.test.ts | 55 ++++++++++++++++++- typescript/packages/lsp/src/run-results.ts | 42 ++++++++++++-- typescript/packages/lsp/src/server.ts | 11 +++- typescript/packages/runner/src/results.ts | 12 +++- typescript/packages/vitest/src/reporter.ts | 30 ++++++++-- typescript/packages/vitest/src/runtime.ts | 36 +++++++++++- .../packages/vitest/tests/reporter.test.ts | 21 ++++++- .../vitest/tests/run-results-wire.test.ts | 20 ++++++- 18 files changed, 351 insertions(+), 31 deletions(-) diff --git a/conformance/adapter/smoke.sh b/conformance/adapter/smoke.sh index a2ec0d73..dbf884fd 100755 --- a/conformance/adapter/smoke.sh +++ b/conformance/adapter/smoke.sh @@ -199,7 +199,7 @@ assert_run_results() { while IFS= read -r oath; do record="$abs/.varar/$oath.json" [ -f "$record" ] || fail "$dir: the run wrote no .varar/$oath.json" "Every adapter persists one run record per oath — the language server reads them." "See doc/adr/0014-run-results-are-a-cross-port-contract.md." - jq -e --arg o "$oath" '.version == 1 and .oathPath == $o and (.sourceHash | startswith("fnv1a:"))' "$record" >/dev/null || fail "$dir: .varar/$oath.json is not the documented payload" "Expected version 1, oathPath \"$oath\", and an fnv1a: sourceHash." "Got: $(jq -c '{version, oathPath, sourceHash}' "$record")" + jq -e --arg o "$oath" '.version == 2 and .oathPath == $o and (.sourceHash | startswith("fnv1a:"))' "$record" >/dev/null || fail "$dir: .varar/$oath.json is not the documented payload" "Expected version 2, oathPath \"$oath\", and an fnv1a: sourceHash." "Got: $(jq -c '{version, oathPath, sourceHash}' "$record")" jq -e '[.examples[].lines[0]] == ([.examples[].lines[0]] | sort)' "$record" >/dev/null || fail "$dir: .varar/$oath.json lists its examples out of document order" "Sort them by first line before writing — the framework's own order is not the oath's." "Got lines: $(jq -c '[.examples[].lines[0]]' "$record")" done <<<"$(oaths_on_disk "$dir")" diff --git a/conformance/run-results/README.md b/conformance/run-results/README.md index 1d38eb9c..af1f464c 100644 --- a/conformance/run-results/README.md +++ b/conformance/run-results/README.md @@ -25,12 +25,22 @@ reads the same payload. ## The value every port builds -Three examples, one per branch of the writer: +Four examples, one per branch of the writer: 1. **passed** — no `failure` key at all. 2. **failed with a mismatch** — `cells` and `anchor` both present, a `£` in the name and a `\n` in the message. 3. **failed by throwing** — `failure` present, `cells` and `anchor` both absent. +4. **failed inside a referenced section** (ADR 0016) — `failure.docPath` names + the oath the failing step was *written* in, and every offset in that failure + (`line`, `cells`, `anchor`) is into **that** document, not this one. Its hash + is the `documents` entry at the top level. `lines` holds only this oath's own + lines: a spliced step contributes none, because its line is not in this file. + +That last one is the reason for `version: 2`. A consumer that places a failure +in the oath named by `oathPath` without checking `docPath` will underline +whatever text happens to sit at those offsets — which is why the field is +pinned here rather than left to each port. `stack` is fixed to `` here. On disk it is runtime-shaped (a V8 stack, a JVM trace, a rendered Rust location) and no consumer parses it — but it is part diff --git a/conformance/run-results/expected.json b/conformance/run-results/expected.json index bf62999e..805620ce 100644 --- a/conformance/run-results/expected.json +++ b/conformance/run-results/expected.json @@ -1,7 +1,13 @@ { - "version": 1, + "version": 2, "oathPath": "varar/library.md", "sourceHash": "fnv1a:1622dfca", + "documents": [ + { + "path": "varar/shared/loans.md", + "sourceHash": "fnv1a:2f0e1d3c" + } + ], "examples": [ { "name": "Maya borrowed *Emma*, due back on June 1, 2026", @@ -47,6 +53,30 @@ "message": "expected the library to refuse", "stack": "" } + }, + { + "name": "An overdue loan blocks a new one", + "status": "failed", + "lines": [ + 20 + ], + "failure": { + "line": 6, + "message": "expected 3 but was 2", + "stack": "", + "cells": [ + { + "from": 41, + "to": 42, + "actual": "2" + } + ], + "anchor": { + "from": 41, + "to": 42 + }, + "docPath": "varar/shared/loans.md" + } } ] } diff --git a/typescript/packages/core/src/execute.ts b/typescript/packages/core/src/execute.ts index eb50beda..5e6f0195 100644 --- a/typescript/packages/core/src/execute.ts +++ b/typescript/packages/core/src/execute.ts @@ -1,6 +1,6 @@ import { CellMismatchError, compareRow, compareTable, ReturnShapeError } from './cell-diff.ts' import { compareDocString } from './doc-string-diff.ts' -import { attachFailureAnchor, failureAnchor } from './failure-anchor.ts' +import { attachFailureAnchor, attachFailureDocPath, failureAnchor } from './failure-anchor.ts' import { compareParams } from './param-diff.ts' import type { ExecutionPlan, PlannedStep } from './plan.ts' import type { Reporter, TestSink } from './ports.ts' @@ -275,9 +275,15 @@ function augmentStack(err: unknown, step: PlannedStep, oathPath: string): unknow // the failing step rather than its whole line. const anchor = failureAnchor(err, step.matchSpan) attachFailureAnchor(err, anchor) + // A step spliced in by a reference block has spans in the document it was + // WRITTEN in, so both the stack frame and the payload must name that file — + // otherwise the frame points an editor at the running oath's line N, which is + // some other sentence entirely. + if (step.docPath !== undefined) attachFailureDocPath(err, step.docPath) + const sourcePath = step.docPath ?? oathPath if (!(err instanceof Error) || typeof err.stack !== 'string') return err const label = step.text.length > 60 ? `${step.text.slice(0, 60)}…` : step.text - const frame = ` at ${label} (${oathPath}:${anchor.startLine}:${anchor.startCol})` + const frame = ` at ${label} (${sourcePath}:${anchor.startLine}:${anchor.startCol})` const lines = err.stack.split('\n') // Find the first existing stack frame (the handler's `.ts` line) and insert // immediately after it. If the error has no frames, fall back to position 1. diff --git a/typescript/packages/core/src/failure-anchor.ts b/typescript/packages/core/src/failure-anchor.ts index 8dfae516..31bba9a8 100644 --- a/typescript/packages/core/src/failure-anchor.ts +++ b/typescript/packages/core/src/failure-anchor.ts @@ -29,3 +29,21 @@ export function readFailureAnchor(error: unknown): Span | undefined { if (typeof error !== 'object' || error === null) return undefined return (error as Record)[ANCHOR] } + +// The document the anchor's offsets belong to, for a step a reference block +// spliced in from another oath (ADR 0016). Travels the same way and for the +// same reason as the anchor: the executor knows the step, and whoever builds +// the failure payload sees only the error. A separate symbol rather than a +// wider anchor payload, so an older copy of the core in the same process still +// reads the anchor it understands. +const DOC_PATH = Symbol.for('varar.failureDocPath') + +export function attachFailureDocPath(error: unknown, docPath: string): void { + if (typeof error !== 'object' || error === null) return + Object.defineProperty(error, DOC_PATH, { value: docPath, enumerable: false, configurable: true }) +} + +export function readFailureDocPath(error: unknown): string | undefined { + if (typeof error !== 'object' || error === null) return undefined + return (error as Record)[DOC_PATH] +} diff --git a/typescript/packages/core/src/failure.ts b/typescript/packages/core/src/failure.ts index d785117d..1543c7a7 100644 --- a/typescript/packages/core/src/failure.ts +++ b/typescript/packages/core/src/failure.ts @@ -1,5 +1,5 @@ import { isCellMismatchError } from './cell-diff.ts' -import { readFailureAnchor } from './failure-anchor.ts' +import { readFailureAnchor, readFailureDocPath } from './failure-anchor.ts' import type { CellFailure, ExampleResult } from './result.ts' // Recover the 1-based failing line from the `:line:col` frame @@ -35,11 +35,17 @@ export function toFailure( // error never passed through a step — then `line` is all a renderer gets. const anchor = readFailureAnchor(error) + // The failing step may have been spliced in from another oath (ADR 0016). Its + // line lives in THAT document's stack frame, and every offset in this payload + // is relative to it. + const docPath = readFailureDocPath(error) + return { - line: failingLine(stack, oathPath) ?? fallbackLine, + line: failingLine(stack, docPath ?? oathPath) ?? fallbackLine, message, stack, ...(cells && cells.length > 0 ? { cells } : {}), ...(anchor ? { anchor: { from: anchor.startOffset, to: anchor.endOffset } } : {}), + ...(docPath !== undefined ? { docPath } : {}), } } diff --git a/typescript/packages/core/src/result.ts b/typescript/packages/core/src/result.ts index d8051a98..1c480780 100644 --- a/typescript/packages/core/src/result.ts +++ b/typescript/packages/core/src/result.ts @@ -25,14 +25,27 @@ export type ExampleResult = { // the same reason `cells` is: a result written by a port (or a release) that // doesn't record it still reads, and falls back to `line`. readonly anchor?: { readonly from: number; readonly to: number } + // The document `line`, `cells` and `anchor` are offsets INTO. Absent — the + // overwhelming majority — means the oath itself. Present only when the + // failing step was spliced in from another oath by a reference block (ADR + // 0016): its spans belong to that document, and a renderer that placed them + // in this one would underline whatever text happened to sit at those + // offsets. Its current hash is in `documents`. + readonly docPath?: string } } // The persisted run result for one oath file. The `.varar/.json` file IS a // serialized OathResults. export type OathResults = { - readonly version: 1 + readonly version: 2 readonly oathPath: string // POSIX separators, relative to cwd readonly sourceHash: string // hashSource(oath source) at run time + // Every OTHER document this run's steps came from — the oaths a reference + // block pulled steps in from (ADR 0016), with their hashes as run. A consumer + // drops a failure whose document has moved on, exactly as it does for the + // oath's own `sourceHash`. Absent when no step was spliced in, which is the + // common case. + readonly documents?: ReadonlyArray<{ readonly path: string; readonly sourceHash: string }> readonly examples: ReadonlyArray } diff --git a/typescript/packages/core/src/run-diagnostics.ts b/typescript/packages/core/src/run-diagnostics.ts index c12b7d4b..7675935a 100644 --- a/typescript/packages/core/src/run-diagnostics.ts +++ b/typescript/packages/core/src/run-diagnostics.ts @@ -31,12 +31,25 @@ function lineRange(source: string, line: number): { from: number; to: number } { export function runResultDiagnostics( results: OathResults, source: string, + // Which document `source` is. Omitted (the usual case) means the oath itself: + // only failures whose offsets are in the oath are projected. Pass a path from + // `results.documents` to project the failures of steps a reference block + // spliced in from THAT oath instead — their offsets are in its source, not + // this one's (ADR 0016). + forDocument?: string, ): ReadonlyArray { - if (hashSource(source) !== results.sourceHash) return [] + const expectedHash = + forDocument === undefined + ? results.sourceHash + : results.documents?.find((d) => d.path === forDocument)?.sourceHash + if (expectedHash === undefined || hashSource(source) !== expectedHash) return [] const out: RunDiagnostic[] = [] for (const ex of results.examples) { if (ex.status !== 'failed' || !ex.failure) continue const f = ex.failure + // A failure belongs to exactly one document: the oath, or the one a + // reference block spliced its failing step in from. + if (f.docPath !== forDocument) continue if (f.cells && f.cells.length > 0) { for (const c of f.cells) { out.push({ diff --git a/typescript/packages/core/tests/failure-step-span.test.ts b/typescript/packages/core/tests/failure-step-span.test.ts index 13207c8e..9b6fe8e1 100644 --- a/typescript/packages/core/tests/failure-step-span.test.ts +++ b/typescript/packages/core/tests/failure-step-span.test.ts @@ -66,7 +66,7 @@ test('the diagnostic underlines the failing step, leaving the passing one alone' const f = await failureOf() const diags = runResultDiagnostics( { - version: 1, + version: 2, oathPath: 'l.md', sourceHash: hashSource(SOURCE), examples: [{ name: 'e', status: 'failed', lines: [3], failure: f }], diff --git a/typescript/packages/core/tests/run-diagnostics.test.ts b/typescript/packages/core/tests/run-diagnostics.test.ts index 60252be5..ab269976 100644 --- a/typescript/packages/core/tests/run-diagnostics.test.ts +++ b/typescript/packages/core/tests/run-diagnostics.test.ts @@ -4,7 +4,7 @@ import type { OathResults } from '../src/result.ts' import { runResultDiagnostics } from '../src/run-diagnostics.ts' function results(source: string, examples: OathResults['examples']): OathResults { - return { version: 1, oathPath: 's.md', sourceHash: hashSource(source), examples } + return { version: 2, oathPath: 's.md', sourceHash: hashSource(source), examples } } test('cell mismatch → one diagnostic per cell with expected/actual message', () => { @@ -117,7 +117,7 @@ test('cells win over the anchor — a mismatched cell is more precise than its s test('stale sourceHash → no diagnostics', () => { const source = 'x 6 y' const r: OathResults = { - version: 1, + version: 2, oathPath: 's.md', sourceHash: 'fnv1a:00000000', examples: [ @@ -137,3 +137,38 @@ test('all-passed results → no diagnostics', () => { const r = results(source, [{ name: 'ok', status: 'passed', lines: [1] }]) expect(runResultDiagnostics(r, source)).toEqual([]) }) + +// ADR 0016: a failure belongs to exactly one document — the oath, or the one a +// reference block spliced its failing step in from. +test('a spliced failure is projected onto its own document, not the running oath', () => { + const shared = 'I shelve 3 books' + const results: OathResults = { + version: 2, + oathPath: 'varar/fees.md', + sourceHash: hashSource('the shelf holds 2 books'), + documents: [{ path: 'varar/shared.md', sourceHash: hashSource(shared) }], + examples: [ + { + name: 'x', + status: 'failed', + lines: [1], + failure: { + line: 1, + message: 'boom', + stack: 's', + anchor: { from: 2, to: 8 }, + docPath: 'varar/shared.md', + }, + }, + ], + } + + expect(runResultDiagnostics(results, 'the shelf holds 2 books')).toEqual([]) + expect(runResultDiagnostics(results, shared, 'varar/shared.md')).toEqual([ + { from: 2, to: 8, message: 'boom' }, + ]) + // The document moved on since the run. + expect(runResultDiagnostics(results, 'I shelve 4 books', 'varar/shared.md')).toEqual([]) + // No such document in this result. + expect(runResultDiagnostics(results, shared, 'varar/other.md')).toEqual([]) +}) diff --git a/typescript/packages/lsp/src/run-results.test.ts b/typescript/packages/lsp/src/run-results.test.ts index 0456b3a1..478162ce 100644 --- a/typescript/packages/lsp/src/run-results.test.ts +++ b/typescript/packages/lsp/src/run-results.test.ts @@ -4,7 +4,7 @@ import { createRunResultsStore, runLspDiagnostics } from './run-results.ts' const SOURCE = 'x 6 y' const OATH: OathResults = { - version: 1, + version: 2, oathPath: 'docs/a.md', sourceHash: hashSource(SOURCE), examples: [ @@ -49,7 +49,7 @@ describe('RunResultsStore', () => { expect( store.ingest( '/root/.varar/x.json', - JSON.stringify({ version: 2, oathPath: 'x', sourceHash: 'h', examples: [] }), + JSON.stringify({ version: 99, oathPath: 'x', sourceHash: 'h', examples: [] }), ), ).toBeNull() expect(store.oathUris()).toEqual([]) @@ -62,3 +62,54 @@ describe('RunResultsStore', () => { expect(store.get('file:///root/docs/a.md')).toBeUndefined() }) }) + +// ADR 0016: a failure inside a section another oath referenced belongs to the +// document it was WRITTEN in, not the oath that ran it. +describe('a failure spliced in from another oath', () => { + const SHARED = 'shelve 3 books' + const REFERRING: OathResults = { + version: 2, + oathPath: 'varar/fees.md', + sourceHash: hashSource('the fee is 50p'), + documents: [{ path: 'varar/shared.md', sourceHash: hashSource(SHARED) }], + examples: [ + { + name: 'the fee is 50p', + status: 'failed', + lines: [1], + failure: { + line: 1, + message: 'boom', + stack: 's', + anchor: { from: 0, to: 6 }, + docPath: 'varar/shared.md', + }, + }, + ], + } + + it('is not projected onto the oath that ran it', () => { + expect(runLspDiagnostics(REFERRING, 'the fee is 50p')).toEqual([]) + }) + + it('is projected onto the document it was written in', () => { + const [d] = runLspDiagnostics(REFERRING, SHARED, 'varar/shared.md') + expect(d?.message).toBe('boom') + expect(d?.range.start).toEqual({ line: 0, character: 0 }) + }) + + it('is dropped when that document has moved on', () => { + expect(runLspDiagnostics(REFERRING, 'shelve 4 books', 'varar/shared.md')).toEqual([]) + }) + + it('reaches the shared oath through the store, which has no result of its own', () => { + const store = createRunResultsStore('file:///w') + store.ingest('/w/.varar/varar/fees.md.json', JSON.stringify(REFERRING)) + + expect(store.get('file:///w/varar/shared.md')).toBeUndefined() + expect(store.resultsFor('file:///w/varar/shared.md')).toEqual([ + { results: REFERRING, forDocument: 'varar/shared.md' }, + ]) + expect(store.oathUris()).toContain('file:///w/varar/shared.md') + }) +}) diff --git a/typescript/packages/lsp/src/run-results.ts b/typescript/packages/lsp/src/run-results.ts index 4b8959b8..bcc1c49c 100644 --- a/typescript/packages/lsp/src/run-results.ts +++ b/typescript/packages/lsp/src/run-results.ts @@ -12,8 +12,12 @@ export type LspDiagnostic = { // Pure: OathResults + current source → LSP diagnostics (0-based positions). // Reuses the core projection; converts each offset range via spanFromOffsets // (1-based span → 0-based LSP), matching the existing parse-diagnostic mapping. -export function runLspDiagnostics(results: OathResults, source: string): LspDiagnostic[] { - return runResultDiagnostics(results, source).map((d) => { +export function runLspDiagnostics( + results: OathResults, + source: string, + forDocument?: string, +): LspDiagnostic[] { + return runResultDiagnostics(results, source, forDocument).map((d) => { const span = spanFromOffsets(source, d.from, d.to) return { severity: 1, // Error @@ -31,7 +35,7 @@ function isOathResults(v: unknown): v is OathResults { if (typeof v !== 'object' || v === null) return false const o = v as Record return ( - o.version === 1 && + (o.version === 1 || o.version === 2) && typeof o.oathPath === 'string' && typeof o.sourceHash === 'string' && Array.isArray(o.examples) @@ -45,6 +49,15 @@ export type RunResultsStore = { // Forget a .varar json (on delete). Returns the oath URI it had mapped, or null. remove(varJsonPath: string): string | null get(oathUri: string): OathResults | undefined + // Every result that has something to say about this URI: the oath's own + // result, plus — for a shared oath whose sections other oaths reference (ADR + // 0016) — each referencing oath's result, tagged with the document path to + // project. A shared oath is not the subject of any result file of its own, so + // without this its failures would never reach the editor. + resultsFor(uri: string): ReadonlyArray<{ + readonly results: OathResults + readonly forDocument?: string + }> oathUris(): ReadonlyArray } @@ -52,6 +65,7 @@ export function createRunResultsStore(rootUri: string): RunResultsStore { const root = rootUri.replace(/\/$/, '') const byUri = new Map() const uriByPath = new Map() // varJsonPath → oathUri, so deletes resolve + const uriFor = (oathPath: string) => `${root}/${oathPath}` return { ingest(varJsonPath, content) { let parsed: unknown @@ -61,7 +75,7 @@ export function createRunResultsStore(rootUri: string): RunResultsStore { return null } if (!isOathResults(parsed)) return null - const oathUri = `${root}/${parsed.oathPath}` + const oathUri = uriFor(parsed.oathPath) byUri.set(oathUri, parsed) uriByPath.set(varJsonPath, oathUri) return oathUri @@ -74,6 +88,24 @@ export function createRunResultsStore(rootUri: string): RunResultsStore { return oathUri }, get: (oathUri) => byUri.get(oathUri), - oathUris: () => [...byUri.keys()], + resultsFor(uri) { + const out: Array<{ results: OathResults; forDocument?: string }> = [] + const own = byUri.get(uri) + if (own) out.push({ results: own }) + for (const results of byUri.values()) { + for (const doc of results.documents ?? []) { + if (uriFor(doc.path) === uri) out.push({ results, forDocument: doc.path }) + } + } + return out + }, + // Referenced documents too: a run that failed inside a shared section must + // light that file up, and a later clean run must clear it. + oathUris: () => [ + ...new Set([ + ...byUri.keys(), + ...[...byUri.values()].flatMap((r) => (r.documents ?? []).map((d) => uriFor(d.path))), + ]), + ], } } diff --git a/typescript/packages/lsp/src/server.ts b/typescript/packages/lsp/src/server.ts index d515f6ab..933c146b 100644 --- a/typescript/packages/lsp/src/server.ts +++ b/typescript/packages/lsp/src/server.ts @@ -176,9 +176,11 @@ export function registerHandlers( async function publishFor(uri: string): Promise { if (!store) return const parse = toParseDiagnostics(uri) + // A URI can be the subject of several results: its own, plus one per oath + // that referenced a section of it (ADR 0016). + const relevant = runResults?.resultsFor(uri) ?? [] let run: LspDiagnostic[] = [] - const results = runResults?.get(uri) - if (results) { + if (relevant.length > 0) { let source = documents.get(uri)?.getText() if (source === undefined) { try { @@ -187,7 +189,10 @@ export function registerHandlers( source = undefined } } - if (source !== undefined) run = runLspDiagnostics(results, source) + if (source !== undefined) { + const text = source + run = relevant.flatMap((r) => runLspDiagnostics(r.results, text, r.forDocument)) + } } void connection.sendDiagnostics({ uri, diagnostics: [...parse, ...run] as Diagnostic[] }) } diff --git a/typescript/packages/runner/src/results.ts b/typescript/packages/runner/src/results.ts index ce472861..68118a25 100644 --- a/typescript/packages/runner/src/results.ts +++ b/typescript/packages/runner/src/results.ts @@ -43,11 +43,21 @@ export function buildOathResults( oathPath: string, source: string, examples: ReadonlyArray, + // The OTHER documents this run's steps came from — the oaths a reference + // block pulled steps in from (ADR 0016), as `path → source`. Their hashes go + // in the payload so a consumer can tell a stale failure from a live one, the + // same way `sourceHash` does for the oath itself. Empty in a project that + // uses no reference blocks. + referencedSources: ReadonlyMap = new Map(), ): OathResults { + const documents = [...referencedSources] + .map(([path, referencedSource]) => ({ path, sourceHash: hashSource(referencedSource) })) + .sort((a, b) => a.path.localeCompare(b.path)) return { - version: 1, + version: 2, oathPath, sourceHash: hashSource(source), + ...(documents.length > 0 ? { documents } : {}), examples: documentOrder(examples), } } diff --git a/typescript/packages/vitest/src/reporter.ts b/typescript/packages/vitest/src/reporter.ts index 83a38fac..f19a7bc2 100644 --- a/typescript/packages/vitest/src/reporter.ts +++ b/typescript/packages/vitest/src/reporter.ts @@ -16,7 +16,7 @@ import { writeOathResults, } from '@varar/runner' import type { Reporter, TestModule } from 'vitest/node' -import { VARAR_BASELINE_META, VARAR_CONSUMED_META } from './runtime.ts' +import { VARAR_BASELINE_META, VARAR_CONSUMED_META, VARAR_DOCUMENTS_META } from './runtime.ts' // Structural shape of the slice of vitest's TestModule API the collector reads. // `meta()` is typed `unknown` so both vitest's real `TestModule` (whose @@ -79,6 +79,22 @@ export function collectBaselines( return byFile } +// The referenced documents each oath's steps were spliced in from (ADR 0016), +// parked by runtime.ts on the same channel as the baseline. Their hashes go in +// the run result so a consumer can tell a stale failure from a live one. +export function collectDocuments( + testModules: ReadonlyArray, +): ReadonlyMap> { + const byFile = new Map>() + for (const m of testModules) { + const documents = (m.meta() as Record | null | undefined)?.[ + VARAR_DOCUMENTS_META + ] as Record | undefined + if (documents) byFile.set(m.moduleId, new Map(Object.entries(documents))) + } + return byFile +} + // Fold this run's derived baselines into the committed lock. Entries for oaths // that did not run are carried over untouched — vitest runs are routinely // filtered (`vitest run varar/library.md`), and a filtered run must not shrink @@ -120,11 +136,17 @@ export class VararResultsReporter implements Reporter { this.writeStderr = options.writeStderr ?? ((s) => void process.stderr.write(s)) } - private writeResults(byFile: ReadonlyMap>): void { + private writeResults( + byFile: ReadonlyMap>, + documentsByFile: ReadonlyMap>, + ): void { for (const [filepath, examples] of byFile) { const oathPath = toOathPath(filepath, this.cwd) const source = readFileSync(filepath, 'utf8') - writeOathResults(this.cwd, buildOathResults(oathPath, source, examples)) + writeOathResults( + this.cwd, + buildOathResults(oathPath, source, examples, documentsByFile.get(filepath)), + ) } } @@ -168,7 +190,7 @@ export class VararResultsReporter implements Reporter { // `BaselineModuleNode` the pure collectors consume. async onTestRunEnd(testModules: ReadonlyArray = []): Promise { const byFile = collectFromModules(testModules) - this.writeResults(byFile) + this.writeResults(byFile, collectDocuments(testModules)) await this.writeBaselines(collectBaselines(testModules), byFile.size > 0) } } diff --git a/typescript/packages/vitest/src/runtime.ts b/typescript/packages/vitest/src/runtime.ts index b48cf8b7..ee19934d 100644 --- a/typescript/packages/vitest/src/runtime.ts +++ b/typescript/packages/vitest/src/runtime.ts @@ -48,6 +48,10 @@ export type CollectPorts = { // freshly derived baseline rather than reusing a stale one. const pendingBaselines = new Map() +// The referenced documents this oath's steps came from, parked beside the +// baseline and attached on the same channel. +const pendingDocuments = new Map>>() + // The key the file-level task meta carries the derived baseline under. The // reporter reads it back through vitest's TestModule.meta(). export const VARAR_BASELINE_META = 'vararBaseline' @@ -59,9 +63,17 @@ export const VARAR_BASELINE_META = 'vararBaseline' // section was consumed. export const VARAR_CONSUMED_META = 'vararConsumed' +// The sources of every OTHER oath this one's steps were spliced in from (ADR +// 0016), parked on the file's task meta for the reporter to hash into the run +// result's `documents`. Absent when nothing was spliced in. +export const VARAR_DOCUMENTS_META = 'vararDocuments' + export type CollectedExample = { readonly name: string // Unique source lines of the example's matched steps, for the reporter. + // Lines in THIS oath. A step a reference block spliced in from another oath + // (ADR 0016) contributes none: its line belongs to that document, and a + // line-wash renderer would otherwise decorate an unrelated sentence here. readonly lines: ReadonlyArray readonly run: () => void | Promise } @@ -83,7 +95,21 @@ export function collectVararExamples( }), } const registry = buildRegistry() - const p = planOath(path, source, registry, runtimeWorkspace(path, source, ports)) + const workspace = runtimeWorkspace(path, source, ports) + const p = planOath(path, source, registry, workspace) + // Hashes for the documents this oath spliced steps in from go in the run + // result, so a consumer can tell a stale failure from a live one. + const spliced = new Set( + p.examples.flatMap((ex) => ex.steps.map((s) => s.docPath).filter((d) => d !== undefined)), + ) + if (spliced.size > 0) { + pendingDocuments.set( + path, + Object.fromEntries( + [...spliced].map((docPath) => [docPath, workspace.docs.get(docPath)?.source ?? '']), + ), + ) + } // Drift reconciliation, split across the process boundary. Detection happens // HERE, against the runtime plan — the same plan every other port reconciles // from (RSpec at describe time, JUnit in its selector resolver). A paragraph @@ -101,7 +127,11 @@ export function collectVararExamples( if (drifts.length === 0) pendingBaselines.set(path, deriveOathBaseline(source, p.doc, p)) const examples = examplesWithRuns(p, contextFactory(), reporter).map(({ example, run }) => ({ name: example.name, - lines: [...new Set(example.steps.map((s) => s.matchSpan.startLine))], + lines: [ + ...new Set( + example.steps.filter((s) => s.docPath === undefined).map((s) => s.matchSpan.startLine), + ), + ], run, })) if (ports.expectedCount !== undefined && examples.length !== ports.expectedCount) { @@ -133,6 +163,8 @@ type TaskContext = { function attachBaseline(ctx: TaskContext, path: string): void { const fileMeta = ctx.task.file?.meta if (!fileMeta) return + const documents = pendingDocuments.get(path) + if (documents && Object.keys(documents).length > 0) fileMeta[VARAR_DOCUMENTS_META] = documents const baseline = pendingBaselines.get(path) if (!baseline) return pendingBaselines.delete(path) diff --git a/typescript/packages/vitest/tests/reporter.test.ts b/typescript/packages/vitest/tests/reporter.test.ts index 794261db..7b607dbe 100644 --- a/typescript/packages/vitest/tests/reporter.test.ts +++ b/typescript/packages/vitest/tests/reporter.test.ts @@ -18,12 +18,31 @@ describe('buildOathResults', () => { test('wraps examples with version, path, and source hash', () => { const r = buildOathResults('docs/a.md', 'src', [passed, failed]) expect(r).toEqual({ - version: 1, + version: 2, oathPath: 'docs/a.md', sourceHash: hashSource('src'), examples: [passed, failed], }) }) + + // ADR 0016: the oaths a reference block pulled steps in from are hashed too, + // so a consumer can tell a stale failure from a live one. + test('records a hash per referenced document, sorted, and omits the key when there are none', () => { + const r = buildOathResults( + 'docs/a.md', + 'src', + [passed], + new Map([ + ['docs/shared/b.md', 'b source'], + ['docs/shared/a.md', 'a source'], + ]), + ) + expect(r.documents).toEqual([ + { path: 'docs/shared/a.md', sourceHash: hashSource('a source') }, + { path: 'docs/shared/b.md', sourceHash: hashSource('b source') }, + ]) + expect(buildOathResults('docs/a.md', 'src', [passed])).not.toHaveProperty('documents') + }) }) describe('collectFromModules', () => { diff --git a/typescript/packages/vitest/tests/run-results-wire.test.ts b/typescript/packages/vitest/tests/run-results-wire.test.ts index 6402af46..ea74449d 100644 --- a/typescript/packages/vitest/tests/run-results-wire.test.ts +++ b/typescript/packages/vitest/tests/run-results-wire.test.ts @@ -10,9 +10,10 @@ import { expect, test } from 'vitest' const EXPECTED = resolve(import.meta.dirname, '../../../../conformance/run-results/expected.json') const results: OathResults = { - version: 1, + version: 2, oathPath: 'varar/library.md', sourceHash: 'fnv1a:1622dfca', + documents: [{ path: 'varar/shared/loans.md', sourceHash: 'fnv1a:2f0e1d3c' }], examples: [ { name: 'Maya borrowed *Emma*, due back on June 1, 2026', status: 'passed', lines: [3, 4] }, { @@ -33,6 +34,23 @@ const results: OathResults = { lines: [8, 9], failure: { line: 9, message: 'expected the library to refuse', stack: '' }, }, + { + // A failure inside a section this oath referenced (ADR 0016): every + // offset here is into varar/shared/loans.md, named by `docPath` and + // hashed in `documents`. `lines` holds only this oath's own lines — the + // spliced step contributes none. + name: 'An overdue loan blocks a new one', + status: 'failed', + lines: [20], + failure: { + line: 6, + message: 'expected 3 but was 2', + stack: '', + cells: [{ from: 41, to: 42, actual: '2' }], + anchor: { from: 41, to: 42 }, + docPath: 'varar/shared/loans.md', + }, + }, ], } From e0d371f81712d30c82c876a016ddf1a357fa7ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 09:59:06 +0100 Subject: [PATCH 02/12] fix(py): a failure inside a referenced section lands in the document it was written in Ports run-result payload v2: failure.docPath names the document line, cells and anchor are offsets into; documents carries a hash per referenced oath that contributed steps; lines holds only the running oath's own lines. The executor's note names the same document, and both adapters pass the referenced sources through to the collector. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- .../packages/core/src/varar_core/execute.py | 15 ++++++-- .../packages/core/src/varar_core/failure.py | 9 +++-- .../core/src/varar_core/failure_anchor.py | 21 ++++++++++++ python/packages/core/src/varar_core/result.py | 32 +++++++++++++++-- .../core/tests/test_run_results_wire.py | 22 +++++++++++- .../pytest/src/varar_pytest/plugin.py | 34 +++++++++++++++++-- python/packages/pytest/tests/test_results.py | 2 +- .../runner/src/varar_runner/results.py | 24 ++++++++++--- .../unittest/src/varar_unittest/__init__.py | 29 ++++++++++++++-- 9 files changed, 171 insertions(+), 17 deletions(-) diff --git a/python/packages/core/src/varar_core/execute.py b/python/packages/core/src/varar_core/execute.py index 57915977..b3fb7da4 100644 --- a/python/packages/core/src/varar_core/execute.py +++ b/python/packages/core/src/varar_core/execute.py @@ -22,7 +22,11 @@ compare_table, ) from varar_core.doc_string_diff import compare_doc_string -from varar_core.failure_anchor import attach_failure_anchor, failure_anchor +from varar_core.failure_anchor import ( + attach_failure_anchor, + attach_failure_doc_path, + failure_anchor, +) from varar_core.param_diff import compare_params from varar_core.plan import ExecutionPlan, PlannedStep from varar_core.span import utf16_slice @@ -400,9 +404,16 @@ def _augment_stack(err: Exception, step: PlannedStep, oath_path: str) -> Excepti # the failing step rather than its whole line. anchor = failure_anchor(err, step.match_span) attach_failure_anchor(err, anchor) + # A step spliced in by a reference block has spans in the document it was + # WRITTEN in, so both the note and the payload must name that file — + # otherwise the note points an editor at the running oath's line N, which is + # some other sentence entirely. + if step.doc_path is not None: + attach_failure_doc_path(err, step.doc_path) + source_path = step.doc_path or oath_path if not isinstance(err, Exception): return err # type: ignore[return-value] label = step.text[:60] + "…" if len(step.text) > 60 else step.text - frame = f" at {label} ({oath_path}:{anchor.start_line}:{anchor.start_col})" + frame = f" at {label} ({source_path}:{anchor.start_line}:{anchor.start_col})" err.add_note(frame) return err diff --git a/python/packages/core/src/varar_core/failure.py b/python/packages/core/src/varar_core/failure.py index 6aafc981..d38c476f 100644 --- a/python/packages/core/src/varar_core/failure.py +++ b/python/packages/core/src/varar_core/failure.py @@ -9,7 +9,7 @@ from typing import Any from varar_core.cell_diff import is_cell_mismatch_error -from varar_core.failure_anchor import read_failure_anchor +from varar_core.failure_anchor import read_failure_anchor, read_failure_doc_path from varar_core.result import AnchorRange, CellFailure, ExampleFailure @@ -54,7 +54,11 @@ def to_failure( if failing: cells = failing - line = _failing_line(stack, oath_path) if stack else None + # The failing step may have been spliced in from another oath (ADR 0016). + # Its line lives in THAT document's stack frame, and every offset in this + # payload is relative to it. + doc_path = read_failure_doc_path(error) + line = _failing_line(stack, doc_path or oath_path) if stack else None # execute_plan attached the anchor when it caught the error, so this is the # failing step's span (or the first mismatched cell's). Absent only when the @@ -71,4 +75,5 @@ def to_failure( if anchor is None else AnchorRange(from_=anchor.start_offset, to=anchor.end_offset) ), + doc_path=doc_path, ) diff --git a/python/packages/core/src/varar_core/failure_anchor.py b/python/packages/core/src/varar_core/failure_anchor.py index 6da4757c..509665df 100644 --- a/python/packages/core/src/varar_core/failure_anchor.py +++ b/python/packages/core/src/varar_core/failure_anchor.py @@ -42,3 +42,24 @@ def read_failure_anchor(error: object) -> Span | None: """The anchor the executor attached, or None if there is none.""" anchor = getattr(error, _ANCHOR_ATTR, None) return anchor if isinstance(anchor, Span) else None + + +# The document the anchor's offsets belong to, for a step a reference block +# spliced in from another oath (ADR 0016). Travels the same way and for the same +# reason as the anchor: the executor knows the step, and whoever builds the +# failure payload sees only the error. +_DOC_PATH_ATTR = "__varar_failure_doc_path__" + + +def attach_failure_doc_path(error: object, doc_path: str) -> None: + """Record on the error itself which document its offsets are into.""" + try: + setattr(error, _DOC_PATH_ATTR, doc_path) + except (AttributeError, TypeError): + pass + + +def read_failure_doc_path(error: object) -> str | None: + """The document path the executor attached, or None for the oath's own.""" + doc_path = getattr(error, _DOC_PATH_ATTR, None) + return doc_path if isinstance(doc_path, str) else None diff --git a/python/packages/core/src/varar_core/result.py b/python/packages/core/src/varar_core/result.py index 329076b6..25c4f13b 100644 --- a/python/packages/core/src/varar_core/result.py +++ b/python/packages/core/src/varar_core/result.py @@ -46,6 +46,21 @@ class ExampleFailure: # Optional for the same reason ``cells`` is: a result written by a port (or # a release) that doesn't record it still reads, and falls back to ``line``. anchor: AnchorRange | None = None + # The document ``line``, ``cells`` and ``anchor`` are offsets INTO. None — + # the overwhelming majority — means the oath itself. Set only when the + # failing step was spliced in from another oath by a reference block (ADR + # 0016): its spans belong to that document, and a renderer that placed them + # in this one would underline whatever text sat at those offsets. + doc_path: str | None = None + + +@dataclass(frozen=True, slots=True) +class ReferencedDocument: + """An oath other than this one that contributed steps to the run, with its + source hash as run (ADR 0016).""" + + path: str + source_hash: str @dataclass(frozen=True, slots=True) @@ -62,10 +77,14 @@ class ExampleResult: class OathResults: """The persisted run result for one oath file (.varar/.json).""" - version: int # always 1 + version: int # always 2 oath_path: str # POSIX separators, relative to cwd source_hash: str # hashSource(oath source) at run time examples: tuple[ExampleResult, ...] + # Every OTHER document this run's steps came from — the oaths a reference + # block pulled steps in from (ADR 0016), with their hashes as run. Empty + # when no step was spliced in, which is the common case. + documents: tuple[ReferencedDocument, ...] = () def to_wire(results: OathResults) -> dict: @@ -86,6 +105,8 @@ def failure(f: ExampleFailure) -> dict: out["cells"] = [cell(c) for c in f.cells] if f.anchor is not None: out["anchor"] = {"from": f.anchor.from_, "to": f.anchor.to} + if f.doc_path is not None: + out["docPath"] = f.doc_path return out def example(e: ExampleResult) -> dict: @@ -94,9 +115,14 @@ def example(e: ExampleResult) -> dict: out["failure"] = failure(e.failure) return out - return { + out: dict = { "version": results.version, "oathPath": results.oath_path, "sourceHash": results.source_hash, - "examples": [example(e) for e in results.examples], } + if results.documents: + out["documents"] = [ + {"path": d.path, "sourceHash": d.source_hash} for d in results.documents + ] + out["examples"] = [example(e) for e in results.examples] + return out diff --git a/python/packages/core/tests/test_run_results_wire.py b/python/packages/core/tests/test_run_results_wire.py index cb600b1a..11131080 100644 --- a/python/packages/core/tests/test_run_results_wire.py +++ b/python/packages/core/tests/test_run_results_wire.py @@ -15,15 +15,19 @@ ExampleFailure, ExampleResult, OathResults, + ReferencedDocument, to_wire, ) EXPECTED = Path(__file__).resolve().parents[4] / "conformance/run-results/expected.json" RESULTS = OathResults( - version=1, + version=2, oath_path="varar/library.md", source_hash="fnv1a:1622dfca", + documents=( + ReferencedDocument(path="varar/shared/loans.md", source_hash="fnv1a:2f0e1d3c"), + ), examples=( ExampleResult( name="Maya borrowed *Emma*, due back on June 1, 2026", @@ -50,6 +54,22 @@ line=9, message="expected the library to refuse", stack="" ), ), + # A failure inside a section this oath referenced (ADR 0016): every + # offset is into varar/shared/loans.md, named by doc_path and hashed in + # documents. lines holds only this oath's own lines. + ExampleResult( + name="An overdue loan blocks a new one", + status="failed", + lines=(20,), + failure=ExampleFailure( + line=6, + message="expected 3 but was 2", + stack="", + cells=(CellFailure(from_=41, to=42, actual="2"),), + anchor=AnchorRange(from_=41, to=42), + doc_path="varar/shared/loans.md", + ), + ), ), ) diff --git a/python/packages/pytest/src/varar_pytest/plugin.py b/python/packages/pytest/src/varar_pytest/plugin.py index 00e11c43..24394845 100644 --- a/python/packages/pytest/src/varar_pytest/plugin.py +++ b/python/packages/pytest/src/varar_pytest/plugin.py @@ -97,6 +97,19 @@ def pytest_unconfigure(config: pytest.Config) -> None: _STASH.pop(id(config), None) +def _referenced_sources(execution_plan, workspace) -> dict[str, str]: + """The sources of every oath this one's steps were spliced in from (ADR + 0016). Their hashes go in the run record, so a consumer can tell a stale + failure from a live one.""" + paths = { + step.doc_path + for example in execution_plan.examples + for step in example.steps + if step.doc_path is not None + } + return {p: doc.source for p in paths if (doc := workspace.docs.get(p)) is not None} + + def _oath_path(path: Path, root: Path) -> str: """The oath's POSIX path relative to the workspace root — its identity in varar.lock.json and in .varar/.json alike.""" @@ -140,6 +153,7 @@ def collect(self): source=source, oath_path=_oath_path(self.path, root), results=results, + referenced_sources=_referenced_sources(execution_plan, workspace), ) # Reconcile drift against varar.lock.json: a clean run records/updates the @@ -183,13 +197,18 @@ def reportinfo(self): class OathItem(pytest.Item): - def __init__(self, *, example, run, source, oath_path, results, **kw): + def __init__( + self, *, example, run, source, oath_path, results, referenced_sources=None, **kw + ): super().__init__(**kw) self._example = example self._run = run self._source = source self._oath_path = oath_path self._results = results + # The OTHER documents this example's steps were spliced in from (ADR + # 0016), so the run record can hash them. + self._referenced_sources = referenced_sources or {} self._token = None def setup(self) -> None: @@ -207,7 +226,16 @@ def runtest(self) -> None: # in hand — pytest's report carries only rendered text by the time the # session ends, and to_failure needs the exception itself to read the # anchor the executor attached to it. - lines = tuple(dict.fromkeys(s.match_span.start_line for s in self._example.steps)) + # Lines in THIS oath. A step a reference block spliced in from another + # oath (ADR 0016) contributes none: its line belongs to that document, + # and a line-wash renderer would decorate an unrelated sentence here. + lines = tuple( + dict.fromkeys( + s.match_span.start_line + for s in self._example.steps + if s.doc_path is None + ) + ) try: self._run() except BaseException as error: @@ -220,12 +248,14 @@ def runtest(self) -> None: lines=lines, failure=to_failure(error, self._oath_path, lines[0] if lines else 0), ), + self._referenced_sources, ) raise self._results.record( self._oath_path, self._source, ExampleResult(name=self._example.name, status="passed", lines=lines), + self._referenced_sources, ) def teardown(self) -> None: diff --git a/python/packages/pytest/tests/test_results.py b/python/packages/pytest/tests/test_results.py index 759bedd1..8f2358cd 100644 --- a/python/packages/pytest/tests/test_results.py +++ b/python/packages/pytest/tests/test_results.py @@ -44,7 +44,7 @@ def test_a_passing_run_writes_the_oath_result(pytester): pytester.runpytest("-q").assert_outcomes(passed=1) results = _results(pytester) - assert results["version"] == 1 + assert results["version"] == 2 assert results["oathPath"] == "features/vault.md" assert results["sourceHash"].startswith("fnv1a:") assert [(e["status"], e["lines"]) for e in results["examples"]] == [("passed", [3])] diff --git a/python/packages/runner/src/varar_runner/results.py b/python/packages/runner/src/varar_runner/results.py index 767a7317..256a1741 100644 --- a/python/packages/runner/src/varar_runner/results.py +++ b/python/packages/runner/src/varar_runner/results.py @@ -12,7 +12,7 @@ from pathlib import Path from varar_core.hash import hash_source -from varar_core.result import ExampleResult, OathResults, to_wire +from varar_core.result import ExampleResult, OathResults, ReferencedDocument, to_wire def result_file_path(root: Path, oath_path: str) -> Path: @@ -57,19 +57,35 @@ class ResultsCollector: def __init__(self) -> None: self._sources: dict[str, str] = {} self._examples: dict[str, list[ExampleResult]] = {} - - def record(self, oath_path: str, source: str, result: ExampleResult) -> None: + # Per oath: the OTHER documents its steps were spliced in from (ADR + # 0016), as path -> source. Their hashes go in the payload so a consumer + # can tell a stale failure from a live one. + self._documents: dict[str, dict[str, str]] = {} + + def record( + self, + oath_path: str, + source: str, + result: ExampleResult, + referenced_sources: dict[str, str] | None = None, + ) -> None: self._sources[oath_path] = source self._examples.setdefault(oath_path, []).append(result) + if referenced_sources: + self._documents.setdefault(oath_path, {}).update(referenced_sources) def write_all(self, root: Path) -> list[Path]: written = [] for oath_path, examples in self._examples.items(): results = OathResults( - version=1, + version=2, oath_path=oath_path, source_hash=hash_source(self._sources[oath_path]), examples=tuple(sorted(examples, key=_document_order)), + documents=tuple( + ReferencedDocument(path=path, source_hash=hash_source(text)) + for path, text in sorted(self._documents.get(oath_path, {}).items()) + ), ) written.append(write_oath_results(root, results)) return written diff --git a/python/packages/unittest/src/varar_unittest/__init__.py b/python/packages/unittest/src/varar_unittest/__init__.py index d709987d..82ec5c46 100644 --- a/python/packages/unittest/src/varar_unittest/__init__.py +++ b/python/packages/unittest/src/varar_unittest/__init__.py @@ -124,6 +124,20 @@ def _oath_test_case( execution_plan = plan_oath(rel, source, loaded.registry, workspace) pairs = examples_with_runs(execution_plan, loaded.create_context, RecordingReporter()) + # The sources of every oath this one's steps were spliced in from (ADR + # 0016). Their hashes go in the run record, so a consumer can tell a stale + # failure from a live one. + referenced = { + path: doc.source + for path in { + step.doc_path + for ex in execution_plan.examples + for step in ex.steps + if step.doc_path is not None + } + if (doc := workspace.docs.get(path)) is not None + } + methods: dict[str, Any] = {"__doc__": rel} seen: dict[str, int] = {} for example, run in pairs: @@ -135,7 +149,9 @@ def _oath_test_case( seen[stem] = idx + 1 display = base if idx == 0 else f"{base}[{idx}]" method_name = f"test_{stem}" if idx == 0 else f"test_{stem}_{idx}" - methods[method_name] = _make_test_method(run, display, source, rel, example, results) + methods[method_name] = _make_test_method( + run, display, source, rel, example, results, referenced + ) # Reconcile drift: a clean run records/updates the baseline; a paragraph # that was an example and no longer matches becomes a failing test method @@ -162,14 +178,23 @@ def _make_test_method( rel_path: str, example: Any, results: ResultsCollector, + referenced_sources: dict[str, str], ) -> Callable[[Any], None]: - lines = tuple(dict.fromkeys(s.match_span.start_line for s in example.steps)) + # Lines in THIS oath. A step a reference block spliced in from another oath + # (ADR 0016) contributes none: its line belongs to that document, and a + # line-wash renderer would decorate an unrelated sentence here. + lines = tuple( + dict.fromkeys( + s.match_span.start_line for s in example.steps if s.doc_path is None + ) + ) def record(status: str, failure: Any = None) -> None: results.record( rel_path, source, ExampleResult(name=example.name, status=status, lines=lines, failure=failure), + referenced_sources, ) def test(self: unittest.TestCase) -> None: From 9e39530a6b96febf19b3bad37a3a2fa55777ad31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 10:00:19 +0100 Subject: [PATCH 03/12] fix(go): a failure inside a referenced section lands in the document it was written in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports run-result payload v2: ExampleFailure.DocPath names the document Line, Cells and Anchor are offsets into; OathResults.Documents carries a hash per referenced oath that contributed steps; Lines holds only the running oath's own lines. attachLocation names the document a spliced step was written in, so ToFailure no longer discards the precise line and anchor when the path differs from the oath — a location for another document now MEANS a spliced step rather than a stray one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- go/core/execute.go | 10 ++++++++- go/core/failure.go | 11 +++++++++- go/core/failure_test.go | 17 ++++++++++----- go/core/result.go | 29 ++++++++++++++++++++----- go/gotest/gotest.go | 39 ++++++++++++++++++++++++++++++++-- go/runner/results.go | 34 +++++++++++++++++++++++++---- go/runner/results_wire_test.go | 21 +++++++++++++++++- 7 files changed, 142 insertions(+), 19 deletions(-) diff --git a/go/core/execute.go b/go/core/execute.go index 7272277c..75a19a3d 100644 --- a/go/core/execute.go +++ b/go/core/execute.go @@ -233,11 +233,19 @@ func tableRows(table Table) Value { func attachLocation(error StepError, step PlannedStep, oathPath string) StepFailure { a := anchor(error, step.MatchSpan) + // A step a reference block spliced in (ADR 0016) has spans in the document + // it was WRITTEN in, so the location must name that file — otherwise a + // renderer points at the running oath's line N, which is some other + // sentence entirely. + path := oathPath + if step.DocPath != "" { + path = step.DocPath + } return StepFailure{ Error: error, Location: &FailureLocation{ Label: truncateLabel(step.Text), - Path: oathPath, + Path: path, Line: a.StartLine, Anchor: AnchorRange{From: a.StartOffset, To: a.EndOffset}, }, diff --git a/go/core/failure.go b/go/core/failure.go index e78108ba..c62203e4 100644 --- a/go/core/failure.go +++ b/go/core/failure.go @@ -93,13 +93,21 @@ func bareFailure(error StepError) StepFailure { func ToFailure(failure StepFailure, oathPath string, fallbackLine int) ExampleFailure { line := fallbackLine var anchor *AnchorRange - if here := failure.Location; here != nil && here.Path == oathPath { + // The location's path is the oath, or — for a step a reference block + // spliced in (ADR 0016) — the document that step was written in. Either way + // its line and anchor are the precise ones; docPath says which file they + // address. + var docPath string + if here := failure.Location; here != nil { line = here.Line // The executor recorded the anchor with the location, so this is the // failing step's span (or the first mismatched cell's) — what a renderer // underlines instead of the whole line. a := here.Anchor anchor = &a + if here.Path != oathPath { + docPath = here.Path + } } var cells []CellFailure @@ -117,6 +125,7 @@ func ToFailure(failure StepFailure, oathPath string, fallbackLine int) ExampleFa Stack: renderStack(failure), Cells: cells, Anchor: anchor, + DocPath: docPath, } } diff --git a/go/core/failure_test.go b/go/core/failure_test.go index 190f22b0..b455e665 100644 --- a/go/core/failure_test.go +++ b/go/core/failure_test.go @@ -50,13 +50,20 @@ func TestToFailureRecordsTheAnchorOfTheStepThatFailed(t *testing.T) { } } -func TestToFailureFallsBackWhenTheLocationIsForAnotherOath(t *testing.T) { +// A location naming a document other than the oath being run is a step a +// reference block spliced in from that document (ADR 0016). Its line and anchor +// are the precise ones — they are simply offsets into that file, which is what +// docPath says. +func TestToFailureRecordsTheDocumentASplicedStepWasWrittenIn(t *testing.T) { f := ToFailure(located(returnShapeError("bad")), "other.md", 99) - if f.Anchor != nil { - t.Errorf("anchor %v, want none for a different oath", *f.Anchor) + if f.DocPath != "l.md" { + t.Errorf("docPath %q, want \"l.md\"", f.DocPath) } - if f.Line != 99 { - t.Errorf("line %d, want the fallback 99", f.Line) + if f.Anchor == nil { + t.Fatal("a spliced failure keeps its anchor — in the other document's offsets") + } + if f.Line != 3 { + t.Errorf("line %d, want the location's line 3, not the fallback", f.Line) } } diff --git a/go/core/result.go b/go/core/result.go index 96d9c9ce..f79df696 100644 --- a/go/core/result.go +++ b/go/core/result.go @@ -43,10 +43,25 @@ type ExampleFailure struct { Stack string `json:"stack"` Cells []CellFailure `json:"cells,omitempty"` Anchor *AnchorRange `json:"anchor,omitempty"` + // DocPath is the document Line, Cells and Anchor are offsets INTO. Empty — + // the overwhelming majority — means the oath itself. Set only when the + // failing step was spliced in from another oath by a reference block (ADR + // 0016): its spans belong to that document, and a renderer that placed them + // in this one would underline whatever text sat at those offsets. + DocPath string `json:"docPath,omitempty"` +} + +// ReferencedDocument is an oath other than this one that contributed steps to +// the run, with its source hash as run (ADR 0016). +type ReferencedDocument struct { + Path string `json:"path"` + SourceHash string `json:"sourceHash"` } // ExampleResult is the run result for one BDD example. Lines are the 1-based -// source lines of its steps (the editor's line-wash anchors). +// source lines of its steps IN THIS OATH (the editor's line-wash anchors) — a +// step spliced in from another oath contributes none, because its line is not +// in this file. type ExampleResult struct { Name string `json:"name"` Status ExampleStatus `json:"status"` @@ -59,8 +74,12 @@ type ExampleResult struct { // HashSource over the oath as it was run, so a reader can tell whether the // offsets still apply to the buffer in front of it. type OathResults struct { - Version int `json:"version"` - OathPath string `json:"oathPath"` - SourceHash string `json:"sourceHash"` - Examples []ExampleResult `json:"examples"` + Version int `json:"version"` + OathPath string `json:"oathPath"` + SourceHash string `json:"sourceHash"` + // Documents holds every OTHER document this run's steps came from — the + // oaths a reference block pulled steps in from (ADR 0016), with their hashes + // as run. Omitted when no step was spliced in, which is the common case. + Documents []ReferencedDocument `json:"documents,omitempty"` + Examples []ExampleResult `json:"examples"` } diff --git a/go/gotest/gotest.go b/go/gotest/gotest.go index 8802efba..6c7cac91 100644 --- a/go/gotest/gotest.go +++ b/go/gotest/gotest.go @@ -43,6 +43,9 @@ type Case struct { // the 1-based source lines of its steps. Empty for a drift case. ExampleName string Lines []int + // The other documents this example's steps were spliced in from (ADR 0016), + // as path→source, for the run record's hashes. + referenced map[string]string } // Collect enumerates every example (and any drift) matched by varar.config.json @@ -96,12 +99,20 @@ func Collect(root string, build BuildRegistry, ctx ContextFactory, update bool) r := rel p := plan example := p.Examples[index] + // Lines in THIS oath. A step a reference block spliced in from + // another oath (ADR 0016) contributes none: its line belongs to that + // document, and a line-wash renderer would decorate an unrelated + // sentence here. var lines []int for _, step := range example.Steps { + if step.DocPath != "" { + continue + } if len(lines) == 0 || lines[len(lines)-1] != step.MatchSpan.StartLine { lines = append(lines, step.MatchSpan.StartLine) } } + refs := referencedSources(p, workspace) cases = append(cases, Case{ Name: r + "::" + display, Source: src, @@ -110,6 +121,7 @@ func Collect(root string, build BuildRegistry, ctx ContextFactory, update bool) run: func() *core.StepFailure { return runner.RunExample(p, ctx, index) }, ExampleName: example.Name, Lines: lines, + referenced: refs, }) } @@ -153,7 +165,7 @@ func Run(t *testing.T, root string, build BuildRegistry, ctx ContextFactory) { if failure == nil { results.Record(c.Rel, c.Source, core.ExampleResult{ Name: c.ExampleName, Status: core.StatusPassed, Lines: c.Lines, - }) + }, c.referenced) return } // Recorded from the failure itself: ToFailure reads the anchor the @@ -167,7 +179,7 @@ func Run(t *testing.T, root string, build BuildRegistry, ctx ContextFactory) { Status: core.StatusFailed, Lines: c.Lines, Failure: ptr(core.ToFailure(*failure, c.Rel, line)), - }) + }, c.referenced) t.Error(runner.RenderFailure(*failure, c.Source, c.Rel)) }) } @@ -201,3 +213,26 @@ func projectWorkspace(oaths []string, root string) core.OathWorkspace { } return core.BuildWorkspace(docs) } + +// referencedSources is the source of every oath this plan's steps were spliced +// in from (ADR 0016). Their hashes go in the run record, so a consumer can tell +// a stale failure from a live one. +func referencedSources(plan core.ExecutionPlan, workspace core.OathWorkspace) map[string]string { + var out map[string]string + for _, example := range plan.Examples { + for _, step := range example.Steps { + if step.DocPath == "" { + continue + } + doc, ok := workspace.Docs[step.DocPath] + if !ok { + continue + } + if out == nil { + out = map[string]string{} + } + out[step.DocPath] = doc.Source + } + } + return out +} diff --git a/go/runner/results.go b/go/runner/results.go index 81164634..7e04dd56 100644 --- a/go/runner/results.go +++ b/go/runner/results.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "sort" "github.com/varar-dev/varar/go/core" ) @@ -71,20 +72,38 @@ type Results struct { order []string sources map[string]string examples map[string][]core.ExampleResult + // Per oath: the other documents its steps were spliced in from (ADR 0016), + // as path→source. + documents map[string]map[string]string } // NewResults is an empty collector. func NewResults() *Results { - return &Results{sources: map[string]string{}, examples: map[string][]core.ExampleResult{}} + return &Results{ + sources: map[string]string{}, + examples: map[string][]core.ExampleResult{}, + documents: map[string]map[string]string{}, + } } -// Record accumulates one example's outcome. -func (r *Results) Record(oathPath, source string, result core.ExampleResult) { +// Record accumulates one example's outcome. referencedSources carries the OTHER +// documents this oath's steps were spliced in from (ADR 0016), as path→source; +// their hashes go in the payload so a consumer can tell a stale failure from a +// live one. Nil in a project that uses no reference blocks. +func (r *Results) Record(oathPath, source string, result core.ExampleResult, referencedSources map[string]string) { if _, seen := r.sources[oathPath]; !seen { r.order = append(r.order, oathPath) } r.sources[oathPath] = source r.examples[oathPath] = append(r.examples[oathPath], result) + if len(referencedSources) > 0 { + if r.documents[oathPath] == nil { + r.documents[oathPath] = map[string]string{} + } + for path, text := range referencedSources { + r.documents[oathPath][path] = text + } + } } // FlushAll writes every oath held, and forgets them. Write errors are ignored @@ -96,14 +115,21 @@ func (r *Results) FlushAll(root string) { if len(examples) == 0 { continue } + var documents []core.ReferencedDocument + for path, text := range r.documents[oathPath] { + documents = append(documents, core.ReferencedDocument{Path: path, SourceHash: core.HashSource(text)}) + } + sort.Slice(documents, func(i, j int) bool { return documents[i].Path < documents[j].Path }) _, _ = WriteOathResults(root, core.OathResults{ - Version: 1, + Version: 2, OathPath: oathPath, SourceHash: core.HashSource(r.sources[oathPath]), + Documents: documents, Examples: examples, }) } r.order = nil r.sources = map[string]string{} r.examples = map[string][]core.ExampleResult{} + r.documents = map[string]map[string]string{} } diff --git a/go/runner/results_wire_test.go b/go/runner/results_wire_test.go index 48cce1f7..812cc2a8 100644 --- a/go/runner/results_wire_test.go +++ b/go/runner/results_wire_test.go @@ -15,9 +15,12 @@ import ( func wireResults() core.OathResults { return core.OathResults{ - Version: 1, + Version: 2, OathPath: "varar/library.md", SourceHash: "fnv1a:1622dfca", + Documents: []core.ReferencedDocument{ + {Path: "varar/shared/loans.md", SourceHash: "fnv1a:2f0e1d3c"}, + }, Examples: []core.ExampleResult{ { Name: "Maya borrowed *Emma*, due back on June 1, 2026", @@ -46,6 +49,22 @@ func wireResults() core.OathResults { Stack: "", }, }, + { + // A failure inside a section this oath referenced (ADR 0016): + // every offset is into varar/shared/loans.md, named by docPath + // and hashed in documents. Lines holds only this oath's own. + Name: "An overdue loan blocks a new one", + Status: core.StatusFailed, + Lines: []int{20}, + Failure: &core.ExampleFailure{ + Line: 6, + Message: "expected 3 but was 2", + Stack: "", + Cells: []core.CellFailure{{From: 41, To: 42, Actual: "2"}}, + Anchor: &core.AnchorRange{From: 41, To: 42}, + DocPath: "varar/shared/loans.md", + }, + }, }, } } From f095e6874f8fe30187b0a3272a4a88e3ef0a54c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 10:01:37 +0100 Subject: [PATCH 04/12] fix(ruby): a failure inside a referenced section lands in the document it was written in Ports run-result payload v2: ExampleFailure#doc_path names the document line, cells and anchor are offsets into; OathResults#documents carries a hash per referenced oath that contributed steps; lines holds only the running oath's own lines. The executor attaches the document path to the error alongside the anchor, and both adapters pass the referenced sources through to the collector. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- ruby/packages/core/lib/varar/core/execute.rb | 4 +++ ruby/packages/core/lib/varar/core/failure.rb | 3 +- .../core/lib/varar/core/failure_anchor.rb | 17 +++++++++++ ruby/packages/core/lib/varar/core/result.rb | 28 +++++++++++++++---- .../spec/varar/core/run_results_wire_spec.rb | 18 +++++++++++- ruby/packages/minitest/lib/varar/minitest.rb | 19 +++++++++++-- ruby/packages/rspec/lib/varar/rspec.rb | 20 +++++++++++-- .../runner/lib/varar/runner/results.rb | 18 ++++++++++-- 8 files changed, 110 insertions(+), 17 deletions(-) diff --git a/ruby/packages/core/lib/varar/core/execute.rb b/ruby/packages/core/lib/varar/core/execute.rb index 8c447e57..984b8815 100644 --- a/ruby/packages/core/lib/varar/core/execute.rb +++ b/ruby/packages/core/lib/varar/core/execute.rb @@ -205,6 +205,10 @@ def observation(ex, example_index, ordinal, file, outcome, error = nil) # so a renderer underlines the step and not its whole line. def augment_stack(error, step, _var_path) FailureAnchor.attach_anchor(error, FailureAnchor.failure_anchor(error, step.match_span)) + # A step spliced in by a reference block has spans in the document it + # was WRITTEN in, so the payload must name that file — otherwise a + # renderer points at the running oath's line N, some other sentence. + FailureAnchor.attach_doc_path(error, step.doc_path) if step.doc_path error end end diff --git a/ruby/packages/core/lib/varar/core/failure.rb b/ruby/packages/core/lib/varar/core/failure.rb index c01884cb..4d976afe 100644 --- a/ruby/packages/core/lib/varar/core/failure.rb +++ b/ruby/packages/core/lib/varar/core/failure.rb @@ -29,7 +29,8 @@ def to_failure(error, _oath_path, fallback_line) message: error.message, stack: render_stack(error), cells: failing_cells(error), - anchor: anchor && AnchorRange.new(from: anchor.start_offset, to: anchor.end_offset) + anchor: anchor && AnchorRange.new(from: anchor.start_offset, to: anchor.end_offset), + doc_path: FailureAnchor.attached_doc_path(error) ) end diff --git a/ruby/packages/core/lib/varar/core/failure_anchor.rb b/ruby/packages/core/lib/varar/core/failure_anchor.rb index c01f06d3..3cedc34b 100644 --- a/ruby/packages/core/lib/varar/core/failure_anchor.rb +++ b/ruby/packages/core/lib/varar/core/failure_anchor.rb @@ -18,6 +18,7 @@ module FailureAnchor # the exception, so it never shows up in `inspect` output the way an # extra attribute would. ANCHOR_IVAR = :@varar_failure_anchor + DOC_PATH_IVAR = :@varar_failure_doc_path def failure_anchor(error, fallback) case error @@ -41,6 +42,22 @@ def attached_anchor(error) error.instance_variable_get(ANCHOR_IVAR) end + + # The document the anchor's offsets belong to, for a step a reference + # block spliced in from another oath (ADR 0016). Travels the same way and + # for the same reason as the anchor: the executor knows the step, and + # whoever builds the failure payload sees only the error. + def attach_doc_path(error, doc_path) + return unless error.respond_to?(:instance_variable_set) + + error.instance_variable_set(DOC_PATH_IVAR, doc_path) + end + + def attached_doc_path(error) + return nil unless error.respond_to?(:instance_variable_get) + + error.instance_variable_get(DOC_PATH_IVAR) + end end end end diff --git a/ruby/packages/core/lib/varar/core/result.rb b/ruby/packages/core/lib/varar/core/result.rb index 2e89b09b..f537bb4c 100644 --- a/ruby/packages/core/lib/varar/core/result.rb +++ b/ruby/packages/core/lib/varar/core/result.rb @@ -21,8 +21,11 @@ module Core # nil when they do not apply, and serialize as absent (not null), so a # reader that predates them still parses the file. `stack` is deliberately # runtime-shaped — no consumer parses it. - ExampleFailure = Data.define(:line, :message, :stack, :cells, :anchor) do - def initialize(line:, message:, stack:, cells: nil, anchor: nil) + # `doc_path` is the document `line`, `cells` and `anchor` are offsets INTO. + # nil — the overwhelming majority — means the oath itself; set only for a + # step a reference block spliced in from another oath (ADR 0016). + ExampleFailure = Data.define(:line, :message, :stack, :cells, :anchor, :doc_path) do + def initialize(line:, message:, stack:, cells: nil, anchor: nil, doc_path: nil) super end end @@ -39,7 +42,15 @@ def initialize(name:, status:, lines:, failure: nil) # separators and is relative to the workspace root; `source_hash` is # Hashing.hash_source over the oath as it was run, so a reader can tell # whether the offsets still apply to the buffer in front of it. - OathResults = Data.define(:version, :oath_path, :source_hash, :examples) + # An oath other than this one that contributed steps to the run, with its + # source hash as run (ADR 0016). + ReferencedDocument = Data.define(:path, :source_hash) + + OathResults = Data.define(:version, :oath_path, :source_hash, :examples, :documents) do + def initialize(version:, oath_path:, source_hash:, examples:, documents: []) + super + end + end # Projection of OathResults onto the JSON shape of .varar/.json. # @@ -51,12 +62,16 @@ module Results module_function def to_wire(results) - { + out = { 'version' => results.version, 'oathPath' => results.oath_path, - 'sourceHash' => results.source_hash, - 'examples' => results.examples.map { |e| example_to_wire(e) } + 'sourceHash' => results.source_hash } + unless results.documents.empty? + out['documents'] = results.documents.map { |d| { 'path' => d.path, 'sourceHash' => d.source_hash } } + end + out['examples'] = results.examples.map { |e| example_to_wire(e) } + out end def example_to_wire(example) @@ -71,6 +86,7 @@ def failure_to_wire(failure) out['cells'] = failure.cells.map { |c| { 'from' => c.from, 'to' => c.to, 'actual' => c.actual } } end out['anchor'] = { 'from' => failure.anchor.from, 'to' => failure.anchor.to } if failure.anchor + out['docPath'] = failure.doc_path if failure.doc_path out end end diff --git a/ruby/packages/core/spec/varar/core/run_results_wire_spec.rb b/ruby/packages/core/spec/varar/core/run_results_wire_spec.rb index 6d791823..cecf3db2 100644 --- a/ruby/packages/core/spec/varar/core/run_results_wire_spec.rb +++ b/ruby/packages/core/spec/varar/core/run_results_wire_spec.rb @@ -17,9 +17,12 @@ module Core let(:results) do OathResults.new( - version: 1, + version: 2, oath_path: 'varar/library.md', source_hash: 'fnv1a:1622dfca', + documents: [ + ReferencedDocument.new(path: 'varar/shared/loans.md', source_hash: 'fnv1a:2f0e1d3c') + ], examples: [ ExampleResult.new( name: 'Maya borrowed *Emma*, due back on June 1, 2026', @@ -42,6 +45,19 @@ module Core failure: ExampleFailure.new( line: 9, message: 'expected the library to refuse', stack: '' ) + ), + # A failure inside a section this oath referenced (ADR 0016): every + # offset is into varar/shared/loans.md, named by doc_path and hashed + # in documents. lines holds only this oath's own lines. + ExampleResult.new( + name: 'An overdue loan blocks a new one', + status: 'failed', lines: [20], + failure: ExampleFailure.new( + line: 6, message: 'expected 3 but was 2', stack: '', + cells: [CellFailure.new(from: 41, to: 42, actual: '2')], + anchor: AnchorRange.new(from: 41, to: 42), + doc_path: 'varar/shared/loans.md' + ) ) ] ) diff --git a/ruby/packages/minitest/lib/varar/minitest.rb b/ruby/packages/minitest/lib/varar/minitest.rb index 417dea62..0a2bdb8e 100644 --- a/ruby/packages/minitest/lib/varar/minitest.rb +++ b/ruby/packages/minitest/lib/varar/minitest.rb @@ -49,6 +49,16 @@ def generate_tests(namespace = Object, root: nil) # Whether a section is a standalone example depends on whether another oath # references it, which is whole-project knowledge (ADR 0016). Built from the # config globs — the full set, for the same reason baseline pruning is. + # The sources of every oath this plan's steps were spliced in from (ADR + # 0016). Their hashes go in the run record, so a consumer can tell a stale + # failure from a live one. + def referenced_sources(plan, workspace) + plan.examples.flat_map { |ex| ex.steps.map(&:doc_path) }.compact.uniq.each_with_object({}) do |path, out| + doc = workspace.docs[path] + out[path] = doc.source if doc + end + end + def project_workspace(oaths, root) docs = oaths.filter_map do |path| Core::Parse.parse(Runner.rel_posix(path, root), File.read(path, encoding: 'UTF-8')) @@ -65,6 +75,7 @@ def build_test_case(oath_path, root, loaded, store, update, results, workspace) # so a relative reference resolves alike and two same-named oaths in # different directories stay distinct (ADR 0016). plan = Runner.plan_oath(rel, source, loaded.registry, workspace) + referenced = referenced_sources(plan, workspace) pairs = Runner.examples_with_runs(plan, loaded.create_context, Runner::RecordingReporter.new) klass = Class.new(::Minitest::Test) @@ -75,7 +86,9 @@ def build_test_case(oath_path, root, loaded, store, update, results, workspace) idx = seen[stem] seen[stem] += 1 method_name = idx.zero? ? "test_#{stem}" : "test_#{stem}_#{idx}" - lines = example.steps.map { |step| step.match_span.start_line }.uniq + # Lines in THIS oath: a step a reference block spliced in from another + # oath (ADR 0016) contributes none, since its line is not in this file. + lines = example.steps.reject(&:doc_path).map { |step| step.match_span.start_line }.uniq klass.define_method(method_name) do run.call rescue StandardError => e @@ -84,14 +97,14 @@ def build_test_case(oath_path, root, loaded, store, update, results, workspace) results.record(rel, source, Core::ExampleResult.new( name: example.name, status: 'failed', lines: lines, failure: Core::Failures.to_failure(e, rel, lines.first || 0) - )) + ), referenced) raise ::Minitest::Assertion, Runner.render_failure(e, source, rel) if Minitest.var_diff_error?(e) raise else results.record(rel, source, Core::ExampleResult.new( name: example.name, status: 'passed', lines: lines, failure: nil - )) + ), referenced) end end diff --git a/ruby/packages/rspec/lib/varar/rspec.rb b/ruby/packages/rspec/lib/varar/rspec.rb index 48aecb4b..4f3209e7 100644 --- a/ruby/packages/rspec/lib/varar/rspec.rb +++ b/ruby/packages/rspec/lib/varar/rspec.rb @@ -46,6 +46,16 @@ def generate(root: nil) # Whether a section is a standalone example depends on whether another oath # references it, which is whole-project knowledge (ADR 0016). Built from the # config globs — the full set, for the same reason baseline pruning is. + # The sources of every oath this plan's steps were spliced in from (ADR + # 0016). Their hashes go in the run record, so a consumer can tell a stale + # failure from a live one. + def referenced_sources(plan, workspace) + plan.examples.flat_map { |ex| ex.steps.map(&:doc_path) }.compact.uniq.each_with_object({}) do |path, out| + doc = workspace.docs[path] + out[path] = doc.source if doc + end + end + def project_workspace(oaths, root) docs = oaths.filter_map do |path| Core::Parse.parse(Runner.rel_posix(path, root), File.read(path, encoding: 'UTF-8')) @@ -62,12 +72,16 @@ def define_group(oath_path, root, loaded, store, update, results, workspace) # so a relative reference resolves alike and two same-named oaths in # different directories stay distinct (ADR 0016). plan = Runner.plan_oath(rel, source, loaded.registry, workspace) + referenced = referenced_sources(plan, workspace) pairs = Runner.examples_with_runs(plan, loaded.create_context, Runner::RecordingReporter.new) drifts = Core::Drifts.reconcile_drift(store, rel, source, plan.doc, plan, update: update) ::RSpec.describe(rel) do pairs.each do |example, run| - lines = example.steps.map { |s| s.match_span.start_line }.uniq + # Lines in THIS oath: a step a reference block spliced in from + # another oath (ADR 0016) contributes none, since its line is not in + # this file. + lines = example.steps.reject(&:doc_path).map { |s| s.match_span.start_line }.uniq # A var diff surfaces as a failure carrying the span-anchored render; # any other exception propagates. RSpec reports both as failures. it(example.name) do @@ -78,14 +92,14 @@ def define_group(oath_path, root, loaded, store, update, results, workspace) results.record(rel, source, Core::ExampleResult.new( name: example.name, status: 'failed', lines: lines, failure: Core::Failures.to_failure(e, rel, lines.first || 0) - )) + ), referenced) raise Runner.render_failure(e, source, rel) if RSpec.var_diff_error?(e) raise else results.record(rel, source, Core::ExampleResult.new( name: example.name, status: 'passed', lines: lines, failure: nil - )) + ), referenced) end end diff --git a/ruby/packages/runner/lib/varar/runner/results.rb b/ruby/packages/runner/lib/varar/runner/results.rb index f39acdd0..a2501374 100644 --- a/ruby/packages/runner/lib/varar/runner/results.rb +++ b/ruby/packages/runner/lib/varar/runner/results.rb @@ -18,6 +18,8 @@ class Results def initialize @sources = {} @examples = {} + # Per oath: the other documents its steps were spliced in from (ADR 0016). + @documents = {} end # `/.varar/.json` — the file the LSP watches. @@ -47,9 +49,15 @@ def self.document_order(examples) # Accumulates one example's outcome; the oath's file is written once its # examples are in. - def record(oath_path, source, result) + # `referenced_sources` carries the OTHER documents this oath's steps were + # spliced in from (ADR 0016), as path => source; their hashes go in the + # payload so a consumer can tell a stale failure from a live one. + def record(oath_path, source, result, referenced_sources = nil) @sources[oath_path] = source (@examples[oath_path] ||= []) << result + return if referenced_sources.nil? || referenced_sources.empty? + + (@documents[oath_path] ||= {}).merge!(referenced_sources) end # Writes what has been recorded for `oath_path` and forgets it. Passing @@ -59,11 +67,15 @@ def flush(root, oath_path) recorded = @examples.delete(oath_path) return nil if recorded.nil? || recorded.empty? + documents = (@documents.delete(oath_path) || {}).sort.map do |path, text| + Core::ReferencedDocument.new(path: path, source_hash: Core::Hash32.hash_source(text)) + end self.class.write(root, Core::OathResults.new( - version: 1, + version: 2, oath_path: oath_path, source_hash: Core::Hash32.hash_source(@sources[oath_path]), - examples: self.class.document_order(recorded) + examples: self.class.document_order(recorded), + documents: documents )) end From 89b4113f8ab29c5ca9775d7306e445b8dbb5bfe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 10:03:41 +0100 Subject: [PATCH 05/12] fix(rust): a failure inside a referenced section lands in the document it was written in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports run-result payload v2: ExampleFailure.doc_path names the document line, cells and anchor are offsets into; OathResults.documents carries a hash per referenced oath that contributed steps; lines holds only the running oath's own lines. attach_location names the document a spliced step was written in, so to_failure no longer discards the precise line and anchor when the path differs from the oath — a location for another document now MEANS a spliced step rather than a stray one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- rust/cargotest/src/lib.rs | 29 ++++++++++++- rust/core/src/execute.rs | 9 +++- rust/core/src/failure.rs | 11 +++-- rust/core/src/result.rs | 53 ++++++++++++++++++++++-- rust/core/tests/failure_test.rs | 14 ++++--- rust/core/tests/run_results_wire_test.rs | 27 +++++++++++- rust/runner/src/results.rs | 42 +++++++++++++++++-- 7 files changed, 167 insertions(+), 18 deletions(-) diff --git a/rust/cargotest/src/lib.rs b/rust/cargotest/src/lib.rs index d0dd875c..a58906f9 100644 --- a/rust/cargotest/src/lib.rs +++ b/rust/cargotest/src/lib.rs @@ -21,6 +21,7 @@ #![allow(clippy::result_large_err)] use std::any::Any; +use std::collections::BTreeMap; use std::path::Path; use std::rc::Rc; use std::sync::{Arc, Mutex}; @@ -133,14 +134,20 @@ fn trials_recording( let (sf, src, r) = (rel.clone(), source.clone(), rel.clone()); let example = &execution.examples[index]; let name = example.name.clone(); + // Lines in THIS oath. A step a reference block spliced in from + // another oath (ADR 0016) contributes none: its line belongs to + // that document, and a line-wash renderer would decorate an + // unrelated sentence here. let mut lines: Vec = example .steps .iter() + .filter(|s| s.doc_path.is_none()) .map(|s| s.match_span.start_line) .collect(); lines.dedup(); let recorder = Arc::clone(results); let ws = Arc::clone(&workspace); + let referenced = referenced_sources(&execution, &workspace); trials.push(Trial::test(format!("{rel}::{display}"), move || { let outcome = run_one_failure(&sf, &src, build_registry, context, index, &ws); let recorded = match &outcome { @@ -162,7 +169,7 @@ fn trials_recording( }, }; if let Ok(mut results) = recorder.lock() { - results.record(&r, &src, recorded); + results.record(&r, &src, recorded, &referenced); } outcome.map_err(|failure| Failed::from(render_failure(&failure, &src, &r))) })); @@ -218,3 +225,23 @@ fn project_workspace(oaths: &[std::path::PathBuf], root: &Path) -> OathWorkspace .collect(); build_workspace(&docs) } + +/// The source of every oath this plan's steps were spliced in from (ADR 0016). +/// Their hashes go in the run record, so a consumer can tell a stale failure +/// from a live one. +fn referenced_sources( + execution: &varar_core::plan::ExecutionPlan, + workspace: &OathWorkspace, +) -> BTreeMap { + let mut out = BTreeMap::new(); + for example in &execution.examples { + for step in &example.steps { + if let Some(path) = &step.doc_path + && let Some(doc) = workspace.docs.get(path) + { + out.insert(path.clone(), doc.source.clone()); + } + } + } + out +} diff --git a/rust/core/src/execute.rs b/rust/core/src/execute.rs index bbcb7692..9cc17959 100644 --- a/rust/core/src/execute.rs +++ b/rust/core/src/execute.rs @@ -321,9 +321,16 @@ fn attach_location(error: StepError, step: &PlannedStep, oath_path: &str) -> Ste let label = truncate_label(&step.text); StepFailure { error, + // A step a reference block spliced in (ADR 0016) has spans in the + // document it was WRITTEN in, so the location must name that file — + // otherwise a renderer points at the running oath's line N, which is + // some other sentence entirely. location: Some(FailureLocation { label, - path: oath_path.to_string(), + path: step + .doc_path + .clone() + .unwrap_or_else(|| oath_path.to_string()), line: anchor.start_line, anchor: AnchorRange { from: anchor.start_offset, diff --git a/rust/core/src/failure.rs b/rust/core/src/failure.rs index 6ad7c4c4..a7a43da0 100644 --- a/rust/core/src/failure.rs +++ b/rust/core/src/failure.rs @@ -22,13 +22,17 @@ pub fn to_failure(failure: &StepFailure, oath_path: &str, fallback_line: i64) -> _ => None, }; - // Structural path match replaces Java's regex-escaped stack-trace scrape. - let here = failure.location.as_ref().filter(|l| l.path == oath_path); + // The location's path is the oath, or — for a step a reference block + // spliced in (ADR 0016) — the document that step was written in. Either way + // its line and anchor are the precise ones; `doc_path` says which file they + // address. + let here = failure.location.as_ref(); let line = here.map_or(fallback_line, |l| l.line as i64); + let doc_path = here.filter(|l| l.path != oath_path).map(|l| l.path.clone()); // The executor recorded the anchor alongside the location, so this is the // failing step's span (or the first mismatched cell's) — what a renderer // underlines instead of the whole line. `None` when the failure carries no - // location for this oath, i.e. it never passed through one of its steps. + // location at all, i.e. it never passed through a step. let anchor = here.map(|l| l.anchor); let stack = render_stack(failure); @@ -38,6 +42,7 @@ pub fn to_failure(failure: &StepFailure, oath_path: &str, fallback_line: i64) -> stack, cells, anchor, + doc_path, } } diff --git a/rust/core/src/result.rs b/rust/core/src/result.rs index 9ae52a2a..b8fa445d 100644 --- a/rust/core/src/result.rs +++ b/rust/core/src/result.rs @@ -52,6 +52,20 @@ pub struct ExampleFailure { pub stack: String, pub cells: Option>, pub anchor: Option, + /// The document `line`, `cells` and `anchor` are offsets INTO. `None` — the + /// overwhelming majority — means the oath itself. Set only when the failing + /// step was spliced in from another oath by a reference block (ADR 0016): + /// its spans belong to that document, and a renderer that placed them in + /// this one would underline whatever text sat at those offsets. + pub doc_path: Option, +} + +/// An oath other than this one that contributed steps to the run, with its +/// source hash as run (ADR 0016). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReferencedDocument { + pub path: String, + pub source_hash: String, } /// The run result for one BDD example. @@ -69,6 +83,10 @@ pub struct OathResults { pub version: u32, pub oath_path: String, pub source_hash: String, + /// Every OTHER document this run's steps came from — the oaths a reference + /// block pulled steps in from (ADR 0016), with their hashes as run. Empty + /// when no step was spliced in, which is the common case. + pub documents: Vec, pub examples: Vec, } @@ -85,6 +103,25 @@ pub fn to_wire_json(results: &OathResults) -> String { field(&mut out, 1, "version", &results.version.to_string(), true); string_field(&mut out, 1, "oathPath", &results.oath_path, true); string_field(&mut out, 1, "sourceHash", &results.source_hash, true); + if !results.documents.is_empty() { + indent(&mut out, 1); + out.push_str("\"documents\": [\n"); + for (i, doc) in results.documents.iter().enumerate() { + indent(&mut out, 2); + out.push_str("{\n"); + string_field(&mut out, 3, "path", &doc.path, true); + string_field(&mut out, 3, "sourceHash", &doc.source_hash, false); + out.push('\n'); + indent(&mut out, 2); + out.push('}'); + if i + 1 < results.documents.len() { + out.push(','); + } + out.push('\n'); + } + indent(&mut out, 1); + out.push_str("],\n"); + } indent(&mut out, 1); out.push_str("\"examples\": "); write_examples(&mut out, &results.examples, 1); @@ -136,7 +173,8 @@ fn write_failure(out: &mut String, failure: &ExampleFailure, depth: usize) { out.push_str("{\n"); field(out, depth + 1, "line", &failure.line.to_string(), true); string_field(out, depth + 1, "message", &failure.message, true); - let has_more = failure.cells.is_some() || failure.anchor.is_some(); + let has_more = + failure.cells.is_some() || failure.anchor.is_some() || failure.doc_path.is_some(); string_field(out, depth + 1, "stack", &failure.stack, has_more); if let Some(cells) = &failure.cells { indent(out, depth + 1); @@ -156,7 +194,7 @@ fn write_failure(out: &mut String, failure: &ExampleFailure, depth: usize) { } indent(out, depth + 1); out.push(']'); - out.push_str(if failure.anchor.is_some() { + out.push_str(if failure.anchor.is_some() || failure.doc_path.is_some() { ",\n" } else { "\n" @@ -168,7 +206,16 @@ fn write_failure(out: &mut String, failure: &ExampleFailure, depth: usize) { field(out, depth + 2, "from", &anchor.from.to_string(), true); field(out, depth + 2, "to", &anchor.to.to_string(), false); indent(out, depth + 1); - out.push_str("}\n"); + out.push('}'); + out.push_str(if failure.doc_path.is_some() { + ",\n" + } else { + "\n" + }); + } + if let Some(doc_path) = &failure.doc_path { + string_field(out, depth + 1, "docPath", doc_path, false); + out.push('\n'); } indent(out, depth); out.push('}'); diff --git a/rust/core/tests/failure_test.rs b/rust/core/tests/failure_test.rs index 2cd0e022..520da836 100644 --- a/rust/core/tests/failure_test.rs +++ b/rust/core/tests/failure_test.rs @@ -69,9 +69,13 @@ fn to_failure_reads_the_failing_line_from_an_injected_location_else_falls_back() } #[test] -fn to_failure_uses_an_exact_oath_path_match() { - // 'aXmd' must not be treated as matching oath path 'a.md' (Java escapes the - // regex dot; Rust compares paths by `==`). - let sf = located(StepError::Handler(HandlerError::new("boom")), "aXmd", 7); - assert_eq!(42, to_failure(&sf, "a.md", 42).line); +fn to_failure_records_the_document_a_spliced_step_was_written_in() { + // A location naming a document other than the oath being run is a step a + // reference block spliced in from that document (ADR 0016). Its line and + // anchor are the precise ones — they are simply offsets into that file, + // which is what `doc_path` says. + let sf = located(StepError::Handler(HandlerError::new("boom")), "shared.md", 7); + let failure = to_failure(&sf, "a.md", 42); + assert_eq!(Some("shared.md".to_string()), failure.doc_path); + assert_eq!(7, failure.line); } diff --git a/rust/core/tests/run_results_wire_test.rs b/rust/core/tests/run_results_wire_test.rs index 07416e87..ff53fbc7 100644 --- a/rust/core/tests/run_results_wire_test.rs +++ b/rust/core/tests/run_results_wire_test.rs @@ -5,14 +5,19 @@ use varar_core::json_value::parse_json_value; use varar_core::result::{ - AnchorRange, CellFailure, ExampleFailure, ExampleResult, OathResults, Status, to_wire_json, + AnchorRange, CellFailure, ExampleFailure, ExampleResult, OathResults, ReferencedDocument, + Status, to_wire_json, }; fn results() -> OathResults { OathResults { - version: 1, + version: 2, oath_path: "varar/library.md".to_string(), source_hash: "fnv1a:1622dfca".to_string(), + documents: vec![ReferencedDocument { + path: "varar/shared/loans.md".to_string(), + source_hash: "fnv1a:2f0e1d3c".to_string(), + }], examples: vec![ ExampleResult { name: "Maya borrowed *Emma*, due back on June 1, 2026".to_string(), @@ -30,6 +35,7 @@ fn results() -> OathResults { stack: "".to_string(), cells: Some(vec![CellFailure::new(71, 77, "£3.00")]), anchor: Some(AnchorRange { from: 60, to: 90 }), + doc_path: None, }), }, ExampleResult { @@ -42,6 +48,23 @@ fn results() -> OathResults { stack: "".to_string(), cells: None, anchor: None, + doc_path: None, + }), + }, + // A failure inside a section this oath referenced (ADR 0016): every + // offset is into varar/shared/loans.md, named by doc_path and hashed + // in documents. `lines` holds only this oath's own lines. + ExampleResult { + name: "An overdue loan blocks a new one".to_string(), + status: Status::Failed, + lines: vec![20], + failure: Some(ExampleFailure { + line: 6, + message: "expected 3 but was 2".to_string(), + stack: "".to_string(), + cells: Some(vec![CellFailure::new(41, 42, "2")]), + anchor: Some(AnchorRange { from: 41, to: 42 }), + doc_path: Some("varar/shared/loans.md".to_string()), }), }, ], diff --git a/rust/runner/src/results.rs b/rust/runner/src/results.rs index babc157c..7b0ee579 100644 --- a/rust/runner/src/results.rs +++ b/rust/runner/src/results.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use varar_core::hash::hash_source; -use varar_core::result::{ExampleResult, OathResults, to_wire_json}; +use varar_core::result::{ExampleResult, OathResults, ReferencedDocument, to_wire_json}; /// `/.varar/.json` — the file the LSP watches. pub fn result_file_path(root: &Path, oath_path: &str) -> PathBuf { @@ -51,6 +51,9 @@ fn document_order(examples: &mut [ExampleResult]) { pub struct Results { sources: BTreeMap, examples: BTreeMap>, + /// Per oath: the other documents its steps were spliced in from (ADR 0016), + /// as path → source. + documents: BTreeMap>, } impl Results { @@ -58,7 +61,17 @@ impl Results { Results::default() } - pub fn record(&mut self, oath_path: &str, source: &str, result: ExampleResult) { + /// Accumulates one example's outcome. `referenced_sources` carries the OTHER + /// documents this oath's steps were spliced in from (ADR 0016), as + /// path → source; their hashes go in the payload so a consumer can tell a + /// stale failure from a live one. + pub fn record( + &mut self, + oath_path: &str, + source: &str, + result: ExampleResult, + referenced_sources: &BTreeMap, + ) { self.sources .entry(oath_path.to_string()) .or_insert_with(|| source.to_string()); @@ -66,6 +79,16 @@ impl Results { .entry(oath_path.to_string()) .or_default() .push(result); + if !referenced_sources.is_empty() { + self.documents + .entry(oath_path.to_string()) + .or_default() + .extend( + referenced_sources + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); + } } /// Writes every oath held, and forgets them. Errors are ignored on purpose: @@ -77,10 +100,23 @@ impl Results { continue; }; document_order(&mut examples); + let documents = self + .documents + .get(&oath_path) + .map(|docs| { + docs.iter() + .map(|(path, text)| ReferencedDocument { + path: path.clone(), + source_hash: hash_source(text), + }) + .collect() + }) + .unwrap_or_default(); let results = OathResults { - version: 1, + version: 2, oath_path: oath_path.clone(), source_hash: hash_source(source), + documents, examples, }; let _ = write_oath_results(root, &results); From 3d68a8404882fd7b97564329c69f92c36e3129bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 10:05:53 +0100 Subject: [PATCH 06/12] fix(dotnet): a failure inside a referenced section lands in the document it was written in Ports run-result payload v2: ExampleFailure.DocPath names the document Line, Cells and Anchor are offsets into; OathResults.Documents carries a hash per referenced oath that contributed steps; Lines holds only the running oath's own lines. The executor attaches the document path to the exception alongside the anchor, and the VSTest adapter passes the referenced sources through to the collector. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- .../Varar.Core.Tests/RunResultsWireTests.cs | 20 +++++++- dotnet/Varar.Core/Execute.cs | 12 +++++ dotnet/Varar.Core/Failure.cs | 5 +- dotnet/Varar.Core/FailureAnchor.cs | 17 +++++++ dotnet/Varar.Core/Result.cs | 22 ++++++++- dotnet/Varar.Core/ResultJson.cs | 20 ++++++++ dotnet/Varar.Runner/Results.cs | 44 ++++++++++++++++-- dotnet/Varar.TestAdapter/VararAdapter.cs | 46 +++++++++++++++++-- 8 files changed, 174 insertions(+), 12 deletions(-) diff --git a/dotnet/Varar.Core.Tests/RunResultsWireTests.cs b/dotnet/Varar.Core.Tests/RunResultsWireTests.cs index 02d8d164..0ddcc7ff 100644 --- a/dotnet/Varar.Core.Tests/RunResultsWireTests.cs +++ b/dotnet/Varar.Core.Tests/RunResultsWireTests.cs @@ -26,7 +26,7 @@ private static string ExpectedPath() } private static OathResults Results() => new( - 1, + 2, "varar/library.md", "fnv1a:1622dfca", [ @@ -49,7 +49,23 @@ private static string ExpectedPath() ExampleStatus.Failed, [8, 9], new ExampleFailure(9, "expected the library to refuse", "")), - ]); + + // A failure inside a section this oath referenced (ADR 0016): every offset is into + // varar/shared/loans.md, named by DocPath and hashed in Documents. Lines holds only + // this oath's own lines. + new ExampleResult( + "An overdue loan blocks a new one", + ExampleStatus.Failed, + [20], + new ExampleFailure( + 6, + "expected 3 but was 2", + "", + ImmutableArray.Create(new CellFailure(41, 42, "2")), + new AnchorRange(41, 42), + "varar/shared/loans.md")), + ], + [new ReferencedDocument("varar/shared/loans.md", "fnv1a:2f0e1d3c")]); [Fact] public void TheWireFormatMatchesTheCrossPortFixture() diff --git a/dotnet/Varar.Core/Execute.cs b/dotnet/Varar.Core/Execute.cs index d41bfbe6..0535fddb 100644 --- a/dotnet/Varar.Core/Execute.cs +++ b/dotnet/Varar.Core/Execute.cs @@ -91,6 +91,13 @@ public static class Execute // failing step's span (or the first mismatched cell's), which Failures.ToFailure // reads back so a renderer underlines the step and not its whole line. FailureAnchor.Attach(err, FailureAnchor.Anchor(err, step.MatchSpan)); + // A step a reference block spliced in has spans in the document it was WRITTEN in + // (ADR 0016), so the payload must name that file. + if (step.DocPath is not null) + { + FailureAnchor.AttachDocPath(err, step.DocPath); + } + observations.Add(new StepObservation(i + 1, "fail", err)); thrown = err; break; @@ -110,6 +117,11 @@ public static class Execute { var err = rowError ?? new CellMismatchError(bad); FailureAnchor.Attach(err, FailureAnchor.Anchor(err, steps[^1].MatchSpan)); + if (steps[^1].DocPath is not null) + { + FailureAnchor.AttachDocPath(err, steps[^1].DocPath!); + } + observations.Add(new StepObservation(steps.Length, "fail", err)); thrown = err; } diff --git a/dotnet/Varar.Core/Failure.cs b/dotnet/Varar.Core/Failure.cs index c55f4ed8..f604e629 100644 --- a/dotnet/Varar.Core/Failure.cs +++ b/dotnet/Varar.Core/Failure.cs @@ -43,6 +43,9 @@ public static ExampleFailure ToFailure(Exception error, string oathPath, int fal Message: error.Message, Stack: error.StackTrace ?? error.Message, Cells: cells, - Anchor: anchor is null ? null : new AnchorRange(anchor.StartOffset, anchor.EndOffset)); + Anchor: anchor is null ? null : new AnchorRange(anchor.StartOffset, anchor.EndOffset), + // The failing step may have been spliced in from another oath (ADR 0016); every offset + // above is then relative to THAT document. + DocPath: FailureAnchor.AttachedDocPath(error)); } } diff --git a/dotnet/Varar.Core/FailureAnchor.cs b/dotnet/Varar.Core/FailureAnchor.cs index b66f23f2..b1c5184b 100644 --- a/dotnet/Varar.Core/FailureAnchor.cs +++ b/dotnet/Varar.Core/FailureAnchor.cs @@ -35,4 +35,21 @@ public static void Attach(Exception? error, Span anchor) /// The anchor the executor attached, or null if there is none. public static Span? Attached(Exception? error) => error?.Data[AnchorKey] as Span; + + private const string DocPathKey = "varar.failureDocPath"; + + /// + /// Records the document the anchor's offsets belong to, for a step a reference block spliced in + /// from another oath (ADR 0016). Travels the same way and for the same reason as the anchor: + /// the executor knows the step, and whoever builds the failure payload sees only the error. + /// + public static void AttachDocPath(Exception? error, string docPath) + { + if (error is not null) + { + error.Data[DocPathKey] = docPath; + } + } + + public static string? AttachedDocPath(Exception? error) => error?.Data[DocPathKey] as string; } diff --git a/dotnet/Varar.Core/Result.cs b/dotnet/Varar.Core/Result.cs index 14688fd8..8eaab952 100644 --- a/dotnet/Varar.Core/Result.cs +++ b/dotnet/Varar.Core/Result.cs @@ -27,12 +27,25 @@ public enum ExampleStatus /// null when they do not apply, and serialize as absent (not null) so a reader that predates /// them still parses the file. Stack is deliberately runtime-shaped — no consumer parses it. /// +/// +/// The document Line, Cells and Anchor are offsets INTO. Null — the +/// overwhelming majority — means the oath itself. Set only when the failing step was spliced in +/// from another oath by a reference block (ADR 0016): its spans belong to that document, and a +/// renderer that placed them in this one would underline whatever text sat at those offsets. +/// public sealed record ExampleFailure( int Line, string Message, string Stack, ImmutableArray? Cells = null, - AnchorRange? Anchor = null); + AnchorRange? Anchor = null, + string? DocPath = null); + +/// +/// An oath other than this one that contributed steps to the run, with its source hash as run +/// (ADR 0016). +/// +public sealed record ReferencedDocument(string Path, string SourceHash); /// /// The run result for one BDD example. Lines are the 1-based source lines of its steps (the @@ -50,8 +63,13 @@ public sealed record ExampleResult( /// the workspace root; SourceHash is over the oath as it was /// run, so a reader can tell whether the offsets still apply to the buffer in front of it. /// +/// +/// Every OTHER document this run's steps came from — the oaths a reference block pulled steps in +/// from (ADR 0016), with their hashes as run. Empty when no step was spliced in. +/// public sealed record OathResults( int Version, string OathPath, string SourceHash, - ImmutableArray Examples); + ImmutableArray Examples, + ImmutableArray Documents = default); diff --git a/dotnet/Varar.Core/ResultJson.cs b/dotnet/Varar.Core/ResultJson.cs index 2da6be0a..22a1f7dd 100644 --- a/dotnet/Varar.Core/ResultJson.cs +++ b/dotnet/Varar.Core/ResultJson.cs @@ -32,6 +32,20 @@ public static string ToWireJson(OathResults results) writer.WriteNumber("version", results.Version); writer.WriteString("oathPath", results.OathPath); writer.WriteString("sourceHash", results.SourceHash); + if (!results.Documents.IsDefaultOrEmpty) + { + writer.WriteStartArray("documents"); + foreach (var document in results.Documents) + { + writer.WriteStartObject(); + writer.WriteString("path", document.Path); + writer.WriteString("sourceHash", document.SourceHash); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + writer.WriteStartArray("examples"); foreach (var example in results.Examples) { @@ -95,6 +109,12 @@ private static void WriteFailure(Utf8JsonWriter writer, ExampleFailure failure) writer.WriteEndObject(); } + // Present only on a step a reference block spliced in from another oath (ADR 0016). + if (failure.DocPath is not null) + { + writer.WriteString("docPath", failure.DocPath); + } + writer.WriteEndObject(); } } diff --git a/dotnet/Varar.Runner/Results.cs b/dotnet/Varar.Runner/Results.cs index 9715d070..6453f1d7 100644 --- a/dotnet/Varar.Runner/Results.cs +++ b/dotnet/Varar.Runner/Results.cs @@ -17,6 +17,10 @@ public sealed class Results private readonly Dictionary sources = new(StringComparer.Ordinal); private readonly Dictionary> examples = new(StringComparer.Ordinal); + // Per oath: the other documents its steps were spliced in from (ADR 0016), as path → source. + private readonly Dictionary> documents = + new(StringComparer.Ordinal); + /// <root>/.varar/<oathPath>.json — the file the LSP watches. public static string ResultFilePath(string root, string oathPath) => Path.Combine(root, ".varar", oathPath.Replace('/', Path.DirectorySeparatorChar) + ".json"); @@ -33,10 +37,33 @@ public static string Write(string root, OathResults results) return out_; } - /// Accumulates one example's outcome; the oath is written once its examples are in. - public void Record(string oathPath, string source, ExampleResult result) + /// + /// Accumulates one example's outcome; the oath is written once its examples are in. + /// carries the OTHER documents this oath's steps were + /// spliced in from (ADR 0016), as path → source; their hashes go in the payload so a consumer + /// can tell a stale failure from a live one. + /// + public void Record( + string oathPath, + string source, + ExampleResult result, + IReadOnlyDictionary? referencedSources = null) { sources[oathPath] = source; + if (referencedSources is { Count: > 0 }) + { + if (!documents.TryGetValue(oathPath, out var docs)) + { + docs = []; + documents[oathPath] = docs; + } + + foreach (var (path, text) in referencedSources) + { + docs[path] = text; + } + } + if (!examples.TryGetValue(oathPath, out var recorded)) { recorded = []; @@ -54,11 +81,22 @@ public void FlushAll(string root) { foreach (var (oathPath, recorded) in examples.ToList()) { + var docs = documents.TryGetValue(oathPath, out var held) + ? held.OrderBy(d => d.Key, StringComparer.Ordinal) + .Select(d => new ReferencedDocument(d.Key, Hash.HashSource(d.Value))) + .ToImmutableArray() + : []; Write( root, - new OathResults(1, oathPath, Hash.HashSource(sources[oathPath]), [.. recorded])); + new OathResults( + 2, + oathPath, + Hash.HashSource(sources[oathPath]), + [.. recorded], + docs)); } examples.Clear(); + documents.Clear(); } } diff --git a/dotnet/Varar.TestAdapter/VararAdapter.cs b/dotnet/Varar.TestAdapter/VararAdapter.cs index 71940250..c54207ae 100644 --- a/dotnet/Varar.TestAdapter/VararAdapter.cs +++ b/dotnet/Varar.TestAdapter/VararAdapter.cs @@ -189,6 +189,7 @@ internal static void Run( var registry = workspace.Registry; var planCache = new Dictionary(StringComparer.Ordinal); + OathWorkspace? oathWorkspace = null; // Run results for the language server (ADR 0014). VSTest reports test by test with no // end-of-run hook, so results accumulate here and are written once this source's // test cases are done. @@ -226,18 +227,30 @@ Value CreateContext(string file) => var result = new TestResult(testCase); try { + // Built once per run: whether a section is a standalone example depends on + // whether another oath references it (ADR 0016). + oathWorkspace ??= ProjectWorkspace( + Discovery.FindOaths(workspace.Config, workspace.Root), workspace.Root); if (!planCache.TryGetValue(oathPath, out var plan)) { plan = RunnerApi.PlanOath( oathPath, File.ReadAllText(Path.Combine(workspace.Root, oathPath)), workspace.Registry, - ProjectWorkspace(Discovery.FindOaths(workspace.Config, workspace.Root), workspace.Root)); + oathWorkspace); planCache[oathPath] = plan; } var example = plan.Examples[index]; - var lines = example.Steps.Select(step => step.MatchSpan.StartLine).Distinct().ToImmutableArray(); + // Lines in THIS oath. A step a reference block spliced in from another oath + // (ADR 0016) contributes none: its line belongs to that document, and a + // line-wash renderer would decorate an unrelated sentence here. + var lines = example.Steps + .Where(step => step.DocPath is null) + .Select(step => step.MatchSpan.StartLine) + .Distinct() + .ToImmutableArray(); + var referenced = ReferencedSources(plan, oathWorkspace); var source = sourceCache.TryGetValue(oathPath, out var cached) ? cached : sourceCache[oathPath] = File.ReadAllText(Path.Combine(workspace.Root, oathPath)); @@ -246,7 +259,11 @@ Value CreateContext(string file) => if (failure is null) { result.Outcome = TestOutcome.Passed; - results.Record(oathPath, source, new ExampleResult(example.Name, ExampleStatus.Passed, lines)); + results.Record( + oathPath, + source, + new ExampleResult(example.Name, ExampleStatus.Passed, lines), + referenced); } else { @@ -261,7 +278,8 @@ Value CreateContext(string file) => example.Name, ExampleStatus.Failed, lines, - Failures.ToFailure(failure, oathPath, lines.Length > 0 ? lines[0] : 0))); + Failures.ToFailure(failure, oathPath, lines.Length > 0 ? lines[0] : 0)), + referenced); } } catch (Exception e) @@ -279,6 +297,26 @@ Value CreateContext(string file) => } /// The built test assembly plus its workspace root (nearest varar.config.json) and registry. + /// + /// The source of every oath this plan's steps were spliced in from (ADR 0016). Their hashes go + /// in the run record, so a consumer can tell a stale failure from a live one. + /// + private static IReadOnlyDictionary ReferencedSources( + ExecutionPlan plan, + OathWorkspace workspace) + { + var out_ = new Dictionary(StringComparer.Ordinal); + foreach (var step in plan.Examples.SelectMany(e => e.Steps)) + { + if (step.DocPath is not null && workspace.Docs.TryGetValue(step.DocPath, out var doc)) + { + out_[step.DocPath] = doc.Source; + } + } + + return out_; + } + /// /// Parses every discovered oath so references resolve and consumed sections are recognised /// (ADR 0016). Parsing runs no step code, so this is cheap. From 0fc68d7dca59e67be5bfcfd64a28bab5a82a6652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 10:08:27 +0100 Subject: [PATCH 07/12] fix(java): a failure inside a referenced section lands in the document it was written in Ports run-result payload v2 to Java and Kotlin, the last of the seven: ExampleFailure.docPath names the document line, cells and anchor are offsets into; OathResults.documents carries a hash per referenced oath that contributed steps; lines holds only the running oath's own lines. The executor's synthetic stack frame names the same document, so Failure.toFailure scrapes the line out of the right frame, and both adapters pass the referenced sources through to the collector. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- .../src/main/java/dev/varar/core/Execute.java | 7 ++- .../src/main/java/dev/varar/core/Failure.java | 7 ++- .../java/dev/varar/core/FailureAnchor.java | 14 ++++++ .../src/main/java/dev/varar/core/Result.java | 49 ++++++++++++++++++- .../dev/varar/core/RunResultsWireTest.java | 19 ++++++- .../dev/varar/junit/ExampleDescriptor.java | 4 ++ .../dev/varar/junit/OathFileDescriptor.java | 24 ++++++++- .../main/kotlin/dev/varar/kotest/OathSpec.kt | 21 +++++++- .../main/java/dev/varar/runner/Results.java | 26 +++++++++- 9 files changed, 161 insertions(+), 10 deletions(-) diff --git a/java/core/src/main/java/dev/varar/core/Execute.java b/java/core/src/main/java/dev/varar/core/Execute.java index a7e61ab2..21eeb5f5 100644 --- a/java/core/src/main/java/dev/varar/core/Execute.java +++ b/java/core/src/main/java/dev/varar/core/Execute.java @@ -496,7 +496,12 @@ private static Throwable augmentStack(Throwable err, Plan.PlannedStep step, Stri // that's what lets a renderer underline the failing step, not its whole line. Span anchor = FailureAnchor.anchor(err, step.matchSpan()); FailureAnchor.attach(err, anchor); - StackTraceElement synthetic = new StackTraceElement("Step", label, oathPath, anchor.startLine()); + // A step a reference block spliced in has spans in the document it was WRITTEN in (ADR + // 0016), so both the synthetic frame and the payload must name that file — otherwise the + // frame points an editor at the running oath's line N, which is some other sentence. + if (step.docPath() != null) FailureAnchor.attachDocPath(err, step.docPath()); + String sourcePath = step.docPath() != null ? step.docPath() : oathPath; + StackTraceElement synthetic = new StackTraceElement("Step", label, sourcePath, anchor.startLine()); StackTraceElement[] original = err.getStackTrace(); StackTraceElement[] augmented = new StackTraceElement[original.length + 1]; augmented[0] = synthetic; diff --git a/java/core/src/main/java/dev/varar/core/Failure.java b/java/core/src/main/java/dev/varar/core/Failure.java index 387824f4..82c6adbf 100644 --- a/java/core/src/main/java/dev/varar/core/Failure.java +++ b/java/core/src/main/java/dev/varar/core/Failure.java @@ -52,14 +52,17 @@ public static Result.ExampleFailure toFailure(Throwable error, String oathPath, if (!failing.isEmpty()) cells = List.copyOf(failing); } - Integer line = failingLine(stack, oathPath); + // The failing step may have been spliced in from another oath (ADR 0016). Its line lives + // in THAT document's injected frame, and every offset in this payload is relative to it. + String docPath = FailureAnchor.attachedDocPath(error); + Integer line = failingLine(stack, docPath != null ? docPath : oathPath); // The executor attached the anchor when it caught the exception, so this is the failing // step's span (or the first mismatched cell's). Null only when the exception never passed // through a step — then the line is all a renderer gets. Span anchor = FailureAnchor.attached(error); Result.AnchorRange range = anchor == null ? null : new Result.AnchorRange(anchor.startOffset(), anchor.endOffset()); - return new Result.ExampleFailure(line != null ? line : fallbackLine, message, stack, cells, range); + return new Result.ExampleFailure(line != null ? line : fallbackLine, message, stack, cells, range, docPath); } /** Recovers the 1-based failing line from an injected {@code ":)"} frame. */ diff --git a/java/core/src/main/java/dev/varar/core/FailureAnchor.java b/java/core/src/main/java/dev/varar/core/FailureAnchor.java index 0167fc25..cdd3c61e 100644 --- a/java/core/src/main/java/dev/varar/core/FailureAnchor.java +++ b/java/core/src/main/java/dev/varar/core/FailureAnchor.java @@ -42,4 +42,18 @@ static void attach(Throwable error, Span anchor) { static Span attached(Throwable error) { return error == null ? null : ATTACHED.get(error); } + + /** + * The document the anchor's offsets belong to, for a step a reference block spliced in from + * another oath (ADR 0016). Travels the same way and for the same reason as the anchor. + */ + private static final Map DOC_PATHS = Collections.synchronizedMap(new WeakHashMap<>()); + + static void attachDocPath(Throwable error, String docPath) { + if (error != null) DOC_PATHS.put(error, docPath); + } + + static String attachedDocPath(Throwable error) { + return error == null ? null : DOC_PATHS.get(error); + } } diff --git a/java/core/src/main/java/dev/varar/core/Result.java b/java/core/src/main/java/dev/varar/core/Result.java index 076ecb21..3ffca985 100644 --- a/java/core/src/main/java/dev/varar/core/Result.java +++ b/java/core/src/main/java/dev/varar/core/Result.java @@ -49,17 +49,36 @@ public record AnchorRange(int from, int to) {} * a step. Optional for the same reason {@code cells} is: a result written without it still * reads, and a renderer falls back to {@code line}. */ - public record ExampleFailure(int line, String message, String stack, List cells, AnchorRange anchor) { + /** + * @param docPath the document {@code line}, {@code cells} and {@code anchor} are offsets INTO. + * Null — the overwhelming majority — means the oath itself. Set only when the failing step + * was spliced in from another oath by a reference block (ADR 0016): its spans belong to + * that document, and a renderer that placed them in this one would underline whatever text + * sat at those offsets. + */ + public record ExampleFailure( + int line, String message, String stack, List cells, AnchorRange anchor, String docPath) { public ExampleFailure { cells = cells == null ? null : List.copyOf(cells); } + /** A failure written by an oath's own step — the overwhelming majority. */ + public ExampleFailure(int line, String message, String stack, List cells, AnchorRange anchor) { + this(line, message, stack, cells, anchor, null); + } + /** A failure with no anchor — the shape producers wrote before anchors were recorded. */ public ExampleFailure(int line, String message, String stack, List cells) { this(line, message, stack, cells, null); } } + /** + * An oath other than this one that contributed steps to the run, with its source hash as run + * (ADR 0016). + */ + public record ReferencedDocument(String path, String sourceHash) {} + /** * The run result for one BDD example. * @@ -74,9 +93,24 @@ public record ExampleResult(String name, Status status, List lines, Exa } /** The persisted run result for one oath file. */ - public record OathResults(int version, String oathPath, String sourceHash, List examples) { + /** + * @param documents every OTHER document this run's steps came from — the oaths a reference + * block pulled steps in from (ADR 0016), with their hashes as run. Empty when no step was + * spliced in, which is the common case. + */ + public record OathResults( + int version, + String oathPath, + String sourceHash, + List examples, + List documents) { public OathResults { examples = List.copyOf(examples); + documents = documents == null ? List.of() : List.copyOf(documents); + } + + public OathResults(int version, String oathPath, String sourceHash, List examples) { + this(version, oathPath, sourceHash, examples, List.of()); } } @@ -91,6 +125,13 @@ public static Map toWire(OathResults results) { out.put("version", results.version()); out.put("oathPath", results.oathPath()); out.put("sourceHash", results.sourceHash()); + if (!results.documents().isEmpty()) { + out.put( + "documents", + results.documents().stream() + .map(d -> (Object) orderedMap("path", d.path(), "sourceHash", d.sourceHash())) + .toList()); + } out.put( "examples", results.examples().stream().map(Result::exampleToWire).toList()); @@ -129,6 +170,10 @@ private static Map failureToWire(ExampleFailure failure) { "to", failure.anchor().to())); } + // Present only on a step a reference block spliced in from another oath (ADR 0016). + if (failure.docPath() != null) { + out.put("docPath", failure.docPath()); + } return out; } diff --git a/java/core/src/test/java/dev/varar/core/RunResultsWireTest.java b/java/core/src/test/java/dev/varar/core/RunResultsWireTest.java index e1330c22..52fc5203 100644 --- a/java/core/src/test/java/dev/varar/core/RunResultsWireTest.java +++ b/java/core/src/test/java/dev/varar/core/RunResultsWireTest.java @@ -20,7 +20,7 @@ class RunResultsWireTest { private static Result.OathResults results() { return new Result.OathResults( - 1, + 2, "varar/library.md", "fnv1a:1622dfca", List.of( @@ -43,7 +43,22 @@ private static Result.OathResults results() { "Noor borrowed *Kindred*", Result.Status.FAILED, List.of(8, 9), - new Result.ExampleFailure(9, "expected the library to refuse", "", null)))); + new Result.ExampleFailure(9, "expected the library to refuse", "", null)), + // A failure inside a section this oath referenced (ADR 0016): every offset + // is into varar/shared/loans.md, named by docPath and hashed in documents. + // lines holds only this oath's own lines. + new Result.ExampleResult( + "An overdue loan blocks a new one", + Result.Status.FAILED, + List.of(20), + new Result.ExampleFailure( + 6, + "expected 3 but was 2", + "", + List.of(new Result.CellFailure(41, 42, "2")), + new Result.AnchorRange(41, 42), + "varar/shared/loans.md"))), + List.of(new Result.ReferencedDocument("varar/shared/loans.md", "fnv1a:2f0e1d3c"))); } @Test diff --git a/java/junit/src/main/java/dev/varar/junit/ExampleDescriptor.java b/java/junit/src/main/java/dev/varar/junit/ExampleDescriptor.java index 63383c2e..fa0f2c65 100644 --- a/java/junit/src/main/java/dev/varar/junit/ExampleDescriptor.java +++ b/java/junit/src/main/java/dev/varar/junit/ExampleDescriptor.java @@ -72,7 +72,11 @@ public OathEngineExecutionContext execute( OathEngineExecutionContext context, DynamicTestExecutor dynamicTestExecutor) throws Exception { OathFileDescriptor fileDescriptor = fileDescriptor(); Runnable run = fileDescriptor.runFor(example); + // Lines in THIS oath. A step a reference block spliced in from another oath (ADR 0016) + // contributes none: its line belongs to that document, and a line-wash renderer would + // decorate an unrelated sentence here. List lines = example.steps().stream() + .filter(step -> step.docPath() == null) .map(step -> step.matchSpan().startLine()) .distinct() .toList(); diff --git a/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java b/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java index f11c6a54..183856ae 100644 --- a/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java +++ b/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java @@ -6,9 +6,12 @@ import dev.varar.runner.Results; import dev.varar.runner.Run; import dev.varar.runner.StepLoader; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.junit.platform.engine.TestSource; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.reporting.ReportEntry; @@ -77,7 +80,26 @@ final class OathFileDescriptor extends AbstractTestDescriptor implements Node referencedSources() { + Map out = new TreeMap<>(); + for (Plan.PlannedExample example : plan.examples()) { + for (Plan.PlannedStep step : example.steps()) { + if (step.docPath() == null) continue; + try { + out.put(step.docPath(), Files.readString(root.resolve(step.docPath()))); + } catch (IOException e) { + // A file that vanished since the run contributes nothing. + } + } + } + return out; } @Override diff --git a/java/kotest/src/main/kotlin/dev/varar/kotest/OathSpec.kt b/java/kotest/src/main/kotlin/dev/varar/kotest/OathSpec.kt index 03be4ee2..64dae2d1 100644 --- a/java/kotest/src/main/kotlin/dev/varar/kotest/OathSpec.kt +++ b/java/kotest/src/main/kotlin/dev/varar/kotest/OathSpec.kt @@ -75,6 +75,16 @@ abstract class OathSpec(root: Path = Path.of(".")) : FunSpec() { val rel = relOf(oathPath) val source = Files.readString(oathPath) val plan = Run.planOath(rel, source, loaded.registry(), workspace) + // The source of every oath this plan's steps were spliced in from (ADR 0016); their + // hashes go in the run record, so a consumer can tell a stale failure from a live one. + val referenced = + plan + .examples() + .flatMap { it.steps() } + .mapNotNull { it.docPath() } + .distinct() + .mapNotNull { path -> workspace.docs()[path]?.let { path to it.source() } } + .toMap() val runs = Run.examplesWithRuns(plan, loaded.createContext(), Run.RecordingReporter()) // Reconcile drift: a clean run records/updates varar.lock.json; a paragraph that was // an example and no longer matches becomes a failing test (accept with -Dvarar.update). @@ -82,7 +92,14 @@ abstract class OathSpec(root: Path = Path.of(".")) : FunSpec() { context(rel) { for (exampleRun in runs) { val example = exampleRun.example() - val lines = example.steps().map { it.matchSpan().startLine() }.distinct() + // Lines in THIS oath: a step a reference block spliced in from another oath + // (ADR 0016) contributes none, since its line is not in this file. + val lines = + example + .steps() + .filter { it.docPath() == null } + .map { it.matchSpan().startLine() } + .distinct() test(example.name()) { try { exampleRun.run().run() @@ -99,6 +116,7 @@ abstract class OathSpec(root: Path = Path.of(".")) : FunSpec() { lines, Failure.toFailure(failure, rel, lines.firstOrNull() ?: 0), ), + referenced, ) // Reuse the runner's span-anchored rendering — never // re-derive failure text in an adapter. @@ -111,6 +129,7 @@ abstract class OathSpec(root: Path = Path.of(".")) : FunSpec() { rel, source, Result.ExampleResult(example.name(), Result.Status.PASSED, lines, null), + referenced, ) } } diff --git a/java/runner/src/main/java/dev/varar/runner/Results.java b/java/runner/src/main/java/dev/varar/runner/Results.java index 26005de3..67b89990 100644 --- a/java/runner/src/main/java/dev/varar/runner/Results.java +++ b/java/runner/src/main/java/dev/varar/runner/Results.java @@ -12,6 +12,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; /** * Persists run results for the language server (ADR 0014) — the shell half of the contract the @@ -26,6 +27,9 @@ public final class Results { private final Map sources = new LinkedHashMap<>(); private final Map> examples = new LinkedHashMap<>(); + /** Per oath: the other documents its steps were spliced in from (ADR 0016), path → source. */ + private final Map> documents = new LinkedHashMap<>(); + /** {@code /.varar/.json} — the file the LSP watches. */ public static Path resultFilePath(Path root, String oathPath) { return root.resolve(".varar").resolve(oathPath + ".json"); @@ -48,8 +52,21 @@ public static Path write(Path root, Result.OathResults results) { /** Accumulates one example's outcome; the oath's file is written once its examples are in. */ public void record(String oathPath, String source, Result.ExampleResult result) { + record(oathPath, source, result, Map.of()); + } + + /** + * As {@link #record(String, String, Result.ExampleResult)}, plus the OTHER documents this + * oath's steps were spliced in from (ADR 0016), as path → source. Their hashes go in the + * payload so a consumer can tell a stale failure from a live one. + */ + public void record( + String oathPath, String source, Result.ExampleResult result, Map referencedSources) { sources.put(oathPath, source); examples.computeIfAbsent(oathPath, k -> new ArrayList<>()).add(result); + if (!referencedSources.isEmpty()) { + documents.computeIfAbsent(oathPath, k -> new TreeMap<>()).putAll(referencedSources); + } } /** @@ -61,7 +78,14 @@ public void flush(Path root, String oathPath) { if (recorded == null || recorded.isEmpty()) { return; } - write(root, new Result.OathResults(1, oathPath, Hash.hashSource(sources.get(oathPath)), List.copyOf(recorded))); + List docs = documents.getOrDefault(oathPath, new TreeMap<>()).entrySet().stream() + .map(e -> new Result.ReferencedDocument(e.getKey(), Hash.hashSource(e.getValue()))) + .toList(); + documents.remove(oathPath); + write( + root, + new Result.OathResults( + 2, oathPath, Hash.hashSource(sources.get(oathPath)), List.copyOf(recorded), docs)); } /** Writes every oath still held — for a runner with no per-file completion hook. */ From 6b5cb54c94b748e768bbf4a52fbf3eef5746c14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 10:10:31 +0100 Subject: [PATCH 08/12] docs(adr): record run-result v2 in ADRs 0014 and 0016 ADR 0014's payload gains docPath and documents for the per-step document identity reference blocks need; ADR 0016's open list loses the gap that kept a failure inside a shared section out of the editor. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MSCLupVART3c5PjiffmahX --- .../0014-run-results-are-a-cross-port-contract.md | 8 +++++++- doc/adr/0016-reuse-is-a-link.md | 15 +++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/doc/adr/0014-run-results-are-a-cross-port-contract.md b/doc/adr/0014-run-results-are-a-cross-port-contract.md index 05306a15..2a2ba50e 100644 --- a/doc/adr/0014-run-results-are-a-cross-port-contract.md +++ b/doc/adr/0014-run-results-are-a-cross-port-contract.md @@ -60,7 +60,13 @@ and the corpus pins the format.** - `sourceHash` is `hashSource(source)` over the oath's bytes as run. The LSP drops every diagnostic when the hash no longer matches the buffer, which is what stops a stale result from pointing at moved text. - - `version` is `1`. + - `version` is `2`. Version 2 adds the per-step document identity reference + blocks need (ADR 0016): `failure.docPath` names the document a failure's + `line`, `cells` and `anchor` are offsets into — absent means the oath + itself — and a top-level `documents` array carries a hash per other oath + whose steps this run spliced in, so a consumer can tell a stale failure + from a live one exactly as `sourceHash` does for the oath. `lines` holds + only the running oath's own lines. 3. **`stack` stays runtime-shaped.** A V8 stack, a JVM stack trace and a Rust rendered location have nothing in common, and no consumer parses it — it is diff --git a/doc/adr/0016-reuse-is-a-link.md b/doc/adr/0016-reuse-is-a-link.md index 7bbde654..6c753084 100644 --- a/doc/adr/0016-reuse-is-a-link.md +++ b/doc/adr/0016-reuse-is-a-link.md @@ -640,14 +640,13 @@ Two adapter-level notes: ### Still open -- **Run-result v2 (ADR 0014).** `docPath` reaches the plan and the plan artifact, - but *not* the persisted `.varar/.json` payload. Until it does, a failure - inside a referenced section is reported to the LSP with spans in that section's - document and a `sourceHash` for the referencing one, so the editor will not - place it. **A mismatch inside a shared section is therefore not yet rendered - correctly in editors** — the run still fails, with the correct message, in - every runner. This is the next piece of work, and it is a cross-port payload - change with its own golden. +- ~~**Run-result v2 (ADR 0014).**~~ **Done** — `.varar/.json` is version 2: + `failure.docPath` names the document its offsets address, `documents` carries a + hash per referenced oath, and `lines` holds only the running oath's own lines. + The LSP publishes a spliced failure against the referenced document's URI (a + shared oath has no result file of its own, so its failures previously had no + way to reach the editor at all). `conformance/run-results/expected.json` grew a + fourth example, which is what gated the other six ports. - **Ambiguous anchors** (deviation 2) are undetected; the lint rule requiring unique headings in a referenced file is not written. - **LSP reference support** — go-to-definition and hover on a reference block — From c84fefc8988c76fff7011734e3086e9f82788e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 12:18:18 +0100 Subject: [PATCH 09/12] fix(vscode): a changed run result republishes every document it speaks for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A .varar/.json speaks for its oath and for every document its steps were spliced in from (ADR 0016), but the file watcher republished only the oath's own URI. A new failure inside a shared section never reached that file until something else republished it, and a clean rerun left the old squiggle behind. The store now reports every URI a result change affects — the oath, the documents the new record names, and the documents the record it replaces named — and the server republishes each. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RQVAKGfMEKPbT919FqAChP --- .../packages/lsp/src/run-results.test.ts | 52 +++++++++++++++++-- typescript/packages/lsp/src/run-results.ts | 29 +++++++---- typescript/packages/lsp/src/server.ts | 15 +++--- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/typescript/packages/lsp/src/run-results.test.ts b/typescript/packages/lsp/src/run-results.test.ts index 478162ce..de50f865 100644 --- a/typescript/packages/lsp/src/run-results.test.ts +++ b/typescript/packages/lsp/src/run-results.test.ts @@ -37,28 +37,28 @@ describe('runLspDiagnostics', () => { describe('RunResultsStore', () => { it('ingests a valid .varar json and keys it by the oath file URI', () => { const store = createRunResultsStore('file:///root') - const uri = store.ingest('/root/.varar/docs/a.md.json', JSON.stringify(OATH)) - expect(uri).toBe('file:///root/docs/a.md') + const uris = store.ingest('/root/.varar/docs/a.md.json', JSON.stringify(OATH)) + expect(uris).toEqual(['file:///root/docs/a.md']) expect(store.get('file:///root/docs/a.md')).toEqual(OATH) expect(store.oathUris()).toEqual(['file:///root/docs/a.md']) }) it('rejects malformed JSON and a wrong version (stores nothing)', () => { const store = createRunResultsStore('file:///root') - expect(store.ingest('/root/.varar/x.json', 'not json')).toBeNull() + expect(store.ingest('/root/.varar/x.json', 'not json')).toEqual([]) expect( store.ingest( '/root/.varar/x.json', JSON.stringify({ version: 99, oathPath: 'x', sourceHash: 'h', examples: [] }), ), - ).toBeNull() + ).toEqual([]) expect(store.oathUris()).toEqual([]) }) it('remove() drops the entry and returns its oath URI', () => { const store = createRunResultsStore('file:///root') store.ingest('/root/.varar/docs/a.md.json', JSON.stringify(OATH)) - expect(store.remove('/root/.varar/docs/a.md.json')).toBe('file:///root/docs/a.md') + expect(store.remove('/root/.varar/docs/a.md.json')).toEqual(['file:///root/docs/a.md']) expect(store.get('file:///root/docs/a.md')).toBeUndefined() }) }) @@ -113,3 +113,45 @@ describe('a failure spliced in from another oath', () => { expect(store.oathUris()).toContain('file:///w/varar/shared.md') }) }) + +// A result file speaks for the documents its steps came from as well as for +// its oath, so the server must republish each of them when it changes — and a +// document the previous record named must be cleared when the new one drops it. +describe('RunResultsStore: which URIs a result change affects', () => { + const REFERRING: OathResults = { + version: 2, + oathPath: 'varar/fees.md', + sourceHash: hashSource('the fee is 50p'), + documents: [{ path: 'varar/shared.md', sourceHash: hashSource('shelve 3 books') }], + examples: [], + } + + it('ingest() names the oath and every document the record (and the one it replaces) names', () => { + const store = createRunResultsStore('file:///w') + expect(store.ingest('/w/.varar/varar/fees.md.json', JSON.stringify(REFERRING))).toEqual([ + 'file:///w/varar/fees.md', + 'file:///w/varar/shared.md', + ]) + // A clean rerun with the reference removed: shared.md still needs its + // stale squiggle cleared, so it is still reported as affected. + const withoutReference = { ...REFERRING, documents: [] } + expect(store.ingest('/w/.varar/varar/fees.md.json', JSON.stringify(withoutReference))).toEqual([ + 'file:///w/varar/fees.md', + 'file:///w/varar/shared.md', + ]) + // And once nothing names it, only the oath itself is affected. + expect(store.ingest('/w/.varar/varar/fees.md.json', JSON.stringify(withoutReference))).toEqual([ + 'file:///w/varar/fees.md', + ]) + }) + + it('remove() names the oath and every document the removed record named', () => { + const store = createRunResultsStore('file:///w') + store.ingest('/w/.varar/varar/fees.md.json', JSON.stringify(REFERRING)) + expect(store.remove('/w/.varar/varar/fees.md.json')).toEqual([ + 'file:///w/varar/fees.md', + 'file:///w/varar/shared.md', + ]) + expect(store.remove('/w/.varar/varar/fees.md.json')).toEqual([]) + }) +}) diff --git a/typescript/packages/lsp/src/run-results.ts b/typescript/packages/lsp/src/run-results.ts index bcc1c49c..db4c1376 100644 --- a/typescript/packages/lsp/src/run-results.ts +++ b/typescript/packages/lsp/src/run-results.ts @@ -43,11 +43,16 @@ function isOathResults(v: unknown): v is OathResults { } export type RunResultsStore = { - // Parse a .varar/.json and key it by its oath's file:// URI. Returns that - // URI, or null if the content is unparseable / the wrong version. - ingest(varJsonPath: string, content: string): string | null - // Forget a .varar json (on delete). Returns the oath URI it had mapped, or null. - remove(varJsonPath: string): string | null + // Parse a .varar/.json and key it by its oath's file:// URI. Returns + // every URI whose diagnostics the record changes — the oath's own, plus each + // document the new record names and each the record it replaces named (ADR + // 0016: a failure inside a shared section lights that file up, and a clean + // rerun must clear it) — or [] if the content is unparseable / the wrong + // version. The oath's own URI is always first. + ingest(varJsonPath: string, content: string): ReadonlyArray + // Forget a .varar json (on delete). Returns the URIs it had a say about — + // the oath's own first, then the documents it named — or [] if unknown. + remove(varJsonPath: string): ReadonlyArray get(oathUri: string): OathResults | undefined // Every result that has something to say about this URI: the oath's own // result, plus — for a shared oath whose sections other oaths reference (ADR @@ -66,26 +71,30 @@ export function createRunResultsStore(rootUri: string): RunResultsStore { const byUri = new Map() const uriByPath = new Map() // varJsonPath → oathUri, so deletes resolve const uriFor = (oathPath: string) => `${root}/${oathPath}` + const documentUris = (results: OathResults | undefined): ReadonlyArray => + (results?.documents ?? []).map((d) => uriFor(d.path)) return { ingest(varJsonPath, content) { let parsed: unknown try { parsed = JSON.parse(content) } catch { - return null + return [] } - if (!isOathResults(parsed)) return null + if (!isOathResults(parsed)) return [] const oathUri = uriFor(parsed.oathPath) + const previous = byUri.get(oathUri) byUri.set(oathUri, parsed) uriByPath.set(varJsonPath, oathUri) - return oathUri + return [...new Set([oathUri, ...documentUris(previous), ...documentUris(parsed)])] }, remove(varJsonPath) { const oathUri = uriByPath.get(varJsonPath) - if (oathUri === undefined) return null + if (oathUri === undefined) return [] + const removed = byUri.get(oathUri) byUri.delete(oathUri) uriByPath.delete(varJsonPath) - return oathUri + return [...new Set([oathUri, ...documentUris(removed)])] }, get: (oathUri) => byUri.get(oathUri), resultsFor(uri) { diff --git a/typescript/packages/lsp/src/server.ts b/typescript/packages/lsp/src/server.ts index 933c146b..1f5faa24 100644 --- a/typescript/packages/lsp/src/server.ts +++ b/typescript/packages/lsp/src/server.ts @@ -144,18 +144,21 @@ export function registerHandlers( for (const change of params.changes) { const path = uriToPath(change.uri) if (!path.includes('/.varar/') || !path.endsWith('.json')) continue - // FileChangeType: 1 Created, 2 Changed, 3 Deleted - const oathUri = change.type === 3 ? runResults.remove(path) : await ingestWatched(path) - if (oathUri) await publishFor(oathUri) + // FileChangeType: 1 Created, 2 Changed, 3 Deleted. A result file speaks + // for its oath AND for every document its steps were spliced in from + // (ADR 0016), so each of those is republished — including a document + // the previous record named and this one no longer does. + const uris = change.type === 3 ? runResults.remove(path) : await ingestWatched(path) + for (const uri of uris) await publishFor(uri) } }) - async function ingestWatched(path: string): Promise { - if (!store || !runResults) return null + async function ingestWatched(path: string): Promise> { + if (!store || !runResults) return [] try { return runResults.ingest(path, await store.fs().read(path)) } catch { - return null + return [] } } From 70603a7363d7d76c9e1dd87feaecde1ccb6ae802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 12:18:18 +0100 Subject: [PATCH 10/12] fix(ts/vitest): referenced-document hashes are attached once per collection, not carried over MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The map handing a file's referenced-document hashes to the reporter was read but never cleared, unlike the baseline beside it. In watch mode an oath whose reference was removed collected again without setting a new entry, so the stale map was attached — and its hashes written — a second time. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RQVAKGfMEKPbT919FqAChP --- typescript/packages/vitest/src/runtime.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/typescript/packages/vitest/src/runtime.ts b/typescript/packages/vitest/src/runtime.ts index ee19934d..d7fda212 100644 --- a/typescript/packages/vitest/src/runtime.ts +++ b/typescript/packages/vitest/src/runtime.ts @@ -164,6 +164,10 @@ function attachBaseline(ctx: TaskContext, path: string): void { const fileMeta = ctx.task.file?.meta if (!fileMeta) return const documents = pendingDocuments.get(path) + // One-shot, like the baseline below: in watch mode an oath whose reference + // was removed collects again without setting an entry, and a stale map + // left here would be attached — and its hashes written — a second time. + pendingDocuments.delete(path) if (documents && Object.keys(documents).length > 0) fileMeta[VARAR_DOCUMENTS_META] = documents const baseline = pendingBaselines.get(path) if (!baseline) return From c3a4f454b515d17ed1a59f744d045e2d641c6b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 12:18:19 +0100 Subject: [PATCH 11/12] fix(java/junit): referenced oath sources are read once per file, not once per example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recordResult() rescanned the plan and reread every referenced oath for each example it recorded — O(examples × references) disk reads per file. The sources are now read once, in before(), under the same ordering guarantee that caches the example runs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RQVAKGfMEKPbT919FqAChP --- .../dev/varar/junit/OathFileDescriptor.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java b/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java index 183856ae..bbd29f8a 100644 --- a/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java +++ b/java/junit/src/main/java/dev/varar/junit/OathFileDescriptor.java @@ -78,16 +78,21 @@ final class OathFileDescriptor extends AbstractTestDescriptor implements Node referencedSources; + /** Records one example's outcome, for {@link #after} to persist. */ void recordResult(Result.ExampleResult result) { - results.record(oathPath, content, result, referencedSources()); + if (referencedSources == null) referencedSources = readReferencedSources(); + results.record(oathPath, content, result, referencedSources); } - /** - * The source of every oath this plan's steps were spliced in from (ADR 0016). Their hashes go - * in the run record, so a consumer can tell a stale failure from a live one. - */ - private Map referencedSources() { + private Map readReferencedSources() { Map out = new TreeMap<>(); for (Plan.PlannedExample example : plan.examples()) { for (Plan.PlannedStep step : example.steps()) { @@ -158,6 +163,7 @@ public OathEngineExecutionContext before(OathEngineExecutionContext context) { // between examples, in or out of document order. Run.RecordingReporter reporter = new Run.RecordingReporter(); exampleRuns = Run.examplesWithRuns(plan, loadedSteps.createContext(), reporter); + referencedSources = readReferencedSources(); publishDiagnostics(context, reporter.diagnostics()); return context; } From b3f79eaa3e6ffbd261e944e5ef726d02b8933ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 14 Sep 2026 12:18:19 +0100 Subject: [PATCH 12/12] chore(website): the playground produces version 2 run results The run-result contract moved to version 2 (ADR 0014); the browser runner and its error fallback still stamped version 1, which no longer matched the public OathResults type. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RQVAKGfMEKPbT919FqAChP --- typescript/packages/website/src/lib/run-oath.ts | 2 +- typescript/packages/website/src/lib/run-worker.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/typescript/packages/website/src/lib/run-oath.ts b/typescript/packages/website/src/lib/run-oath.ts index 83d3c271..9193ce4b 100644 --- a/typescript/packages/website/src/lib/run-oath.ts +++ b/typescript/packages/website/src/lib/run-oath.ts @@ -74,7 +74,7 @@ export async function runRegisteredOath( executePlan(toRun, { sink, reporter: { diagnostic() {} }, createContext }) await Promise.all(pending) const results: OathResults = { - version: 1, + version: 2, oathPath: oathPath, sourceHash: hashSource(varSource), examples: out, diff --git a/typescript/packages/website/src/lib/run-worker.ts b/typescript/packages/website/src/lib/run-worker.ts index 53ec25d4..7e28a653 100644 --- a/typescript/packages/website/src/lib/run-worker.ts +++ b/typescript/packages/website/src/lib/run-worker.ts @@ -97,7 +97,7 @@ self.onmessage = async (e: MessageEvent) => { } catch (err) { const e2 = err as Error results = { - version: 1, + version: 2, oathPath: input.oathPath, sourceHash: hashSource(input.varSource), examples: [