From 1e3430ec13022309c6e4c5de18c77f794b7ce519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:05:49 +0200 Subject: [PATCH 1/5] feat(testing): a Tests area streams per-case progress, bounds every case, and gets a console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node's Tests area ran every case and rendered ONE frame at the end, so a case that called an external service live (or hung) left the page on "Rendering …" for the whole suite, with nothing to tell slow from stuck and nothing naming the case. - MeshTestRunner streams: every case pending at once, the running case with its ticking elapsed time and its output lines as written, finished cases ✔/✖ — then ONE verdict frame (✅/❌, "N/M passed"). Progress frames carry AreaFrameClassifier.TestsRunningId and never the verdict glyphs, so no consumer can read one as a verdict. - MeshTestCase + Area(host, suite, cases): the listed-cases shape (static methods) runs through the same streaming runner; synchronous bodies run on the Tests I/O pool, so a body that never returns fails "timed out: no verdict within Ns" instead of freezing the render. - AreaProbe (the plugin gate) treats a progress frame as transient; a cut-off run names its progress. - memex tests : reads the area until the verdict, printing each case as it changes. - Column titles and progress title localized (en/de). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/MeshWeaver.Cli/Program.cs | 26 ++ src/MeshWeaver.Cli/TestsCommand.cs | 188 ++++++++++ .../Data/Architecture/DecentralisedTests.md | 3 +- .../Data/Architecture/WritingTests.md | 37 ++ src/MeshWeaver.Layout/AreaFrameClassifier.cs | 36 +- .../Localization/strings.de.json | 8 +- .../Localization/strings.en.json | 8 +- src/MeshWeaver.Testing.InMesh/MeshTestCase.cs | 50 +++ .../MeshTestRunner.Progress.cs | 323 ++++++++++++++++++ .../MeshTestRunner.cs | 66 ++-- test/MeshWeaver.Cli.Test/TestsCommandTest.cs | 66 ++++ .../AreaProbeTest.cs | 26 ++ .../MeshTestRunnerTests.cs | 61 ++++ tools/MeshWeaver.PluginTester/AreaProbe.cs | 15 + 14 files changed, 886 insertions(+), 27 deletions(-) create mode 100644 src/MeshWeaver.Cli/TestsCommand.cs create mode 100644 src/MeshWeaver.Testing.InMesh/MeshTestCase.cs create mode 100644 src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs create mode 100644 test/MeshWeaver.Cli.Test/TestsCommandTest.cs diff --git a/src/MeshWeaver.Cli/Program.cs b/src/MeshWeaver.Cli/Program.cs index bb52f193d1..a04c3c4b58 100644 --- a/src/MeshWeaver.Cli/Program.cs +++ b/src/MeshWeaver.Cli/Program.cs @@ -165,6 +165,32 @@ async Task Run( root.Subcommands.Add(cmd); } +// --- tests ----------------------------------------------------------------- +{ + var pathArg = new Argument("path") { Description = "Node whose Tests area to run (e.g. @Admin/Maintenance/x)." }; + var timeoutOpt = new Option("--timeout") { Description = "Seconds to wait for the verdict.", DefaultValueFactory = _ => 600 }; + var intervalOpt = new Option("--interval") { Description = "Seconds between reads of the area.", DefaultValueFactory = _ => 2 }; + var cmd = new Command("tests", "Run a node's Tests area and stream its progress until the verdict (exit 0 = all passed, 1 = a failure, 4 = no verdict in time).") + { pathArg, timeoutOpt, intervalOpt }; + cmd.SetAction(async (result, ct) => + { + try + { + var cfg = MemexConfig.Resolve(result.GetValue(baseUrlOpt), result.GetValue(tokenOpt)); + using var client = new MemexClient(cfg); + return await TestsCommand.Run(client, result.GetValue(pathArg)!, + TimeSpan.FromSeconds(result.GetValue(timeoutOpt)), TimeSpan.FromSeconds(Math.Max(1, result.GetValue(intervalOpt))), + Console.Out, ct); + } + catch (MemexCliException ex) + { + await Console.Error.WriteLineAsync(ex.Message); + return 2; + } + }); + root.Subcommands.Add(cmd); +} + // --- upload ---------------------------------------------------------------- { var pathArg = new Argument("path") { Description = "Target mesh path {nodePath}/{collection}/{filePath}." }; diff --git a/src/MeshWeaver.Cli/TestsCommand.cs b/src/MeshWeaver.Cli/TestsCommand.cs new file mode 100644 index 0000000000..6a157448dc --- /dev/null +++ b/src/MeshWeaver.Cli/TestsCommand.cs @@ -0,0 +1,188 @@ +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace MeshWeaver.Cli; + +/// +/// memex tests <path> — run (= render) one node's Tests area and print its +/// progress as a console, line by line, until the verdict frame arrives. +/// +/// The Tests area IS the runner (MeshTestRunner): rendering it executes the cases, and +/// the area streams a progress frame per second — every case pending, running (with its elapsed +/// time and its output so far) or finished — then ONE verdict frame. This command reads the area +/// through the same get @path/area/Tests verb an agent uses, prints every row whose state or +/// output changed since the last read, and exits on the verdict: 0 when every counted case +/// passed, 1 when any failed, 4 when no verdict arrived within --timeout (the +/// last progress is printed, so a hung case is NAMED rather than waited on). +/// +/// A Tests area that predates the streaming runner renders only its verdict, so the first read +/// simply prints the finished table — the command works against every portal, streaming or not. +/// +public static class TestsCommand +{ + /// The UiControl.Id every progress frame carries (AreaFrameClassifier.TestsRunningId). + public const string TestsRunningId = "tests-running"; + + private static readonly Regex PassSummary = new(@"(\d+)\s*/\s*(\d+)\s+passed", RegexOptions.CultureInvariant); + + /// One case as the area renders it. + /// The case's name. + /// Its status glyph / verdict. + /// Its elapsed time. + /// Its failure message and output lines. + public sealed record Row(string Case, string Result, string Time, string Output); + + /// What one read of the area says. + /// True for a progress frame — the verdict is still to come. + /// The frame's title (progress or verdict), when it has one. + /// The cases the frame lists as grid rows (empty for a legacy markdown table). + /// Every other string the Tests area carries (a legacy table's markdown). + public sealed record Frame(bool Running, string? Title, IReadOnlyList Rows, IReadOnlyList Text) + { + /// True when the verdict frame says every counted case passed. + public bool Passed + { + get + { + var all = Rows.Select(r => r.Result).Concat(Text).Append(Title ?? ""); + if (all.Any(s => s.Contains('❌'))) + return false; + var summary = all.Select(s => PassSummary.Match(s)).FirstOrDefault(m => m.Success); + return summary is null + ? Rows.Count > 0 || all.Any(s => s.Contains('✅')) + : summary.Groups[1].Value == summary.Groups[2].Value; + } + } + } + + /// + /// Reads a get @path/area/Tests answer. Pure over the JSON: the chrome areas the framework + /// writes into every subscription ($Menu…, $Banner) are skipped, exactly as the + /// plugin gate skips them. + /// + /// The rendered area store. + public static Frame Parse(string json) + { + using var doc = JsonDocument.Parse(json); + var rows = new List(); + var text = new List(); + string? title = null; + var running = false; + if (doc.RootElement.ValueKind == JsonValueKind.Object + && doc.RootElement.TryGetProperty("areas", out var areas) + && areas.ValueKind == JsonValueKind.Object) + { + foreach (var area in areas.EnumerateObject()) + { + if (area.Name.TrimStart('"').StartsWith('$')) + continue; + Walk(area.Value); + } + } + return new Frame(running, title, rows, text); + + void Walk(JsonElement e) + { + switch (e.ValueKind) + { + case JsonValueKind.Object: + if (e.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String + && id.GetString() == TestsRunningId) + running = true; + if (e.TryGetProperty("case", out var c) && e.TryGetProperty("result", out var r)) + { + rows.Add(new Row(Str(c), Str(r), Str(e, "time"), Str(e, "output"))); + return; + } + foreach (var p in e.EnumerateObject()) + Walk(p.Value); + break; + case JsonValueKind.Array: + foreach (var item in e.EnumerateArray()) + Walk(item); + break; + case JsonValueKind.String: + var s = e.GetString() ?? ""; + if (title is null && (s.Contains(" tests — ", StringComparison.Ordinal) || s.Contains("-Tests — ", StringComparison.Ordinal))) + title = StripTags(s); + else if (s.Contains('✅') || s.Contains('❌') || PassSummary.IsMatch(s)) + text.Add(s); + break; + } + } + } + + private static string Str(JsonElement e) => e.ValueKind == JsonValueKind.String ? e.GetString() ?? "" : e.ToString(); + + private static string Str(JsonElement e, string property) => + e.TryGetProperty(property, out var v) ? Str(v) : ""; + + private static string StripTags(string s) => Regex.Replace(s, "<[^>]+>", "").Trim(); + + /// + /// The lines to print for given what was already printed: a row that is + /// new, or whose result or output changed. A row whose only change is its ticking time is not + /// reprinted — the console would scroll with nothing new in it. + /// + /// The rows as last printed, by case name. + /// The frame just read. + public static IReadOnlyList Changes(IDictionary printed, Frame next) + { + var lines = new List(); + foreach (var row in next.Rows) + { + if (printed.TryGetValue(row.Case, out var before) + && before.Result == row.Result && before.Output == row.Output) + continue; + printed[row.Case] = row; + lines.Add($"{row.Result,-10} {row.Case}{(row.Time.Length > 0 ? $" ({row.Time})" : "")}{(row.Output.Length > 0 ? $" — {row.Output}" : "")}"); + } + return lines; + } + + /// Reads the area until the verdict, printing each change; returns the exit code. + /// The portal client. + /// The node whose Tests area to run. + /// How long to wait for the verdict. + /// How often to read the area. + /// Where the console goes. + /// Cancels the wait. + public static async Task Run(MemexClient client, string path, TimeSpan timeout, TimeSpan interval, TextWriter output, CancellationToken ct) + { + var area = $"@{path.TrimStart('@').TrimEnd('/')}/area/Tests"; + var printed = new Dictionary(StringComparer.Ordinal); + var deadline = DateTimeOffset.UtcNow + timeout; + string? lastTitle = null; + while (true) + { + var body = await client.Get(area, ct); + if (body.StartsWith("Error:", StringComparison.Ordinal) || body.StartsWith("Not found", StringComparison.Ordinal)) + { + await output.WriteLineAsync(body); + return 1; + } + var frame = Parse(body); + if (frame.Title is { } title && title != lastTitle) + { + await output.WriteLineAsync($"== {title}"); + lastTitle = title; + } + foreach (var line in Changes(printed, frame)) + await output.WriteLineAsync(line); + if (!frame.Running) + { + foreach (var line in frame.Text) + await output.WriteLineAsync(line); + return frame.Passed ? 0 : 1; + } + if (DateTimeOffset.UtcNow >= deadline) + { + var stuck = printed.Values.Where(r => r.Result == "▶").Select(r => r.Case).ToList(); + await output.WriteLineAsync( + $"no verdict within {timeout.TotalSeconds:F0}s — still running: {(stuck.Count > 0 ? string.Join(", ", stuck) : "(no case reported running)")}"); + return 4; + } + await Task.Delay(interval, ct); + } + } +} diff --git a/src/MeshWeaver.Documentation/Data/Architecture/DecentralisedTests.md b/src/MeshWeaver.Documentation/Data/Architecture/DecentralisedTests.md index 3fe0833bac..ad899089a2 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/DecentralisedTests.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/DecentralisedTests.md @@ -336,7 +336,8 @@ differently. Conflating them is the main way this design goes wrong. `PluginGateRunner` + `AreaProbe`: stands up a mesh, imports content, renders each type's `Tests` layout area over the ordinary client sync stream, and classifies the frames -(`AreaProbe.ClassifyTestsFrame`): compile-progress and *"Area not found"* are transient; `❌` is red; +(`AreaProbe.ClassifyTestsFrame`): compile-progress, a streamed Tests progress frame +(`tests-running`, see [Writing Tests](../WritingTests)) and *"Area not found"* are transient; `❌` is red; `N/M passed` is green iff `N == M`; **no verdict inside the timeout is RED** — *"a Tests area that reports nothing is a broken gate, never a silent pass."* diff --git a/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md b/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md index 91099d9cc0..a78482872d 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md @@ -144,6 +144,43 @@ are counted as `needs-mesh` and run by the gate, seeded from the build's output. `MeshWeaver.Fixture` and the two TestBase assemblies are this repo's OWN test support: they live under `test/` and are never packed or published. +### A `Tests` area streams its progress — and `memex tests` is its console + +Rendering a node's `Tests` area RUNS its cases (`MeshWeaver.Testing.InMesh.MeshTestRunner`). The +area used to emit **one frame, at the end**, so a case that called an external service live — or a +case that hung — left the page on *"Rendering …"* for the whole suite, with nothing to tell a slow +case from a stuck one and nothing naming the case. It now streams: + +| Frame | Shows | Carries | +|---|---|---| +| first, at once | every case ⏳ pending | `Id = tests-running` | +| once a second while a case runs | the running case ▶ with its ticking elapsed time and every output line it has written so far; finished cases ✔ / ✖ | `Id = tests-running` | +| last | the verdict: title ` tests — N/M passed`, rows ✅ / ❌ / ⏭ with time and output | no id | + +- **Every case is bounded.** A case past its bound fails as `timed out: no verdict within Ns` and the + run moves on; a synchronous case runs as one leaf on the mesh's `Tests` I/O pool, so a body that + never returns costs its own bound, never the render thread. +- **The verdict glyphs are reserved for the verdict frame.** A progress frame shows ✔ / ✖ and counts + cases *done*, never ✅ / ❌ or *"N/M passed"* — and it carries + `AreaFrameClassifier.TestsRunningId`, which the plugin gate (`AreaProbe`) and + `AreaFrameClassifier.IsTransientFrame` treat as *keep waiting*. A gate that latched a progress frame + would green a suite whose later cases had not run yet; a run cut off by the gate's timeout reports + the last progress it saw. +- **Two ways to list cases.** `[MeshFact]` classes: `MeshTestRunner.Area(host, suite, assembly)`. + Static methods: `MeshTestRunner.Area(host, suite, cases)` over `MeshTestCase.Of(name, method)` + (synchronous) and `MeshTestCase.Live(name, () => observable, timeout)` (hosted, passes on first + emission); either overload hands the body a line writer whose output streams into the row. +- **From a terminal or an agent:** `memex tests @` reads the area every two seconds, prints each + case as its state or output changes, and exits `0` (all passed), `1` (a failure) or `4` (no verdict + within `--timeout`, naming the case still running). An agent does the same with + `get @/area/Tests`, which answers with the frame the area has reached. 🚨 Rendering RUNS the + suite, so what a second read shows depends on whether it reuses the first read's subscription: + measured once on memex.meshweaver.cloud, a read of `Admin/Maintenance/…/Tests` answered the + finished verdict in 0.9 s for a suite whose live cases alone take 18 s or more — i.e. it reused a + subscription that had already run. A read that opens a NEW subscription (another replica, an + evicted stream) renders the area anew and so runs the suite — live cases included — again; that + half is inferred from the render path, not measured. Treat each fresh subscription as a fresh run. + ## The Canonical Test Base Every monolith test inherits `MonolithMeshTestBase`. The shape is always the same: diff --git a/src/MeshWeaver.Layout/AreaFrameClassifier.cs b/src/MeshWeaver.Layout/AreaFrameClassifier.cs index fc5968d130..5e8bdc3484 100644 --- a/src/MeshWeaver.Layout/AreaFrameClassifier.cs +++ b/src/MeshWeaver.Layout/AreaFrameClassifier.cs @@ -88,6 +88,22 @@ public static class AreaFrameClassifier /// public const string StorageUnavailableId = "storage-unavailable"; + /// + /// of every PROGRESS frame a Tests area serves while its + /// cases are still running (MeshTestRunner.Area): the table shows each case pending, + /// running or finished, with its elapsed time and its output so far. TRANSIENT. + /// + /// The SIXTH state. A Tests area used to emit exactly ONE frame, at the end, so a case + /// that called a slow external service kept the page on "Rendering …" for its whole budget, + /// indistinguishable from a hung one. Streaming the progress fixes the page, and this id is + /// what keeps the stream safe for a consumer that reads a VERDICT off the first frame it can + /// classify (the plugin gate's AreaProbe): a progress frame is a promise that the + /// verdict frame follows, never a verdict of its own — its rows already carry ✔ for a case + /// that passed so far, and read as a verdict that would green a suite whose later cases have + /// not run yet. + /// + public const string TestsRunningId = "tests-running"; + // The pre-id signal, kept as a fallback so a frame that lost its id on the way here (an // older peer, a control rebuilt from partial JSON) is still recognised. Never localize: // BuildNotFoundControl is deliberately English — it is a framework diagnostic, not UI copy. @@ -141,16 +157,26 @@ public static bool IsHubRecycling(UiControl? control) public static bool IsStorageUnavailable(UiControl? control) => HasFrameId(control, StorageUnavailableId); + /// + /// True for a PROGRESS frame of a Tests area whose cases are still running — the + /// verdict frame replaces it once the last case finished or timed out. + /// + /// The rendered control, or null. + public static bool IsTestsRunning(UiControl? control) + => HasFrameId(control, TestsRunningId); + /// /// True for a frame that is not the area's content and will be REPLACED without anyone - /// acting: the compile-progress page, and the it emits once - /// the build settles. The single predicate a waiter needs — "keep waiting, this is not the - /// answer". A genuinely missing area () is deliberately NOT - /// transient: nothing is going to replace it. + /// acting: the compile-progress page, the it emits once + /// the build settles, and a Tests area's progress frame, which its verdict frame replaces. + /// The single predicate a waiter needs — "keep waiting, this is not the answer". A genuinely + /// missing area () is deliberately NOT transient: nothing is + /// going to replace it. /// /// The rendered control, or null. public static bool IsTransientFrame(UiControl? control) - => IsCompileProgress(control) || IsHubRecycling(control) || control is RedirectControl; + => IsCompileProgress(control) || IsHubRecycling(control) || IsTestsRunning(control) + || control is RedirectControl; // UiControl.Id is `object?`, so a frame that came back over the sync stream carries it as // whatever the deserializer produced for a JSON string (a JsonElement, not a string). Compare diff --git a/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json b/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json index 12d528d7a4..88f7482efc 100644 --- a/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json +++ b/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json @@ -1716,5 +1716,11 @@ "ui.threadQueueStateWoken": "Geweckt", "ui.threadQueueStateRetried": "Neu gestartet", "ui.threadQueueStateActionFailed": "Nicht ausführbar", - "ui.threadQueueStateFailed": "Fehlgeschlagen" + "ui.threadQueueStateFailed": "Fehlgeschlagen", + "tests.progress": "{0}-Tests — {1} von {2} erledigt, seit {3}s in Arbeit", + "tests.column.class": "Klasse", + "tests.column.case": "Fall", + "tests.column.result": "Ergebnis", + "tests.column.time": "Zeit", + "tests.column.output": "Ausgabe" } diff --git a/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json b/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json index 8b28ff1dd8..946953d853 100644 --- a/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json +++ b/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json @@ -1716,5 +1716,11 @@ "ui.threadQueueStateWoken": "Woken", "ui.threadQueueStateRetried": "Relaunched", "ui.threadQueueStateActionFailed": "Could not act", - "ui.threadQueueStateFailed": "Failed" + "ui.threadQueueStateFailed": "Failed", + "tests.progress": "{0} tests — {1} of {2} done, running for {3}s", + "tests.column.class": "Class", + "tests.column.case": "Case", + "tests.column.result": "Result", + "tests.column.time": "Time", + "tests.column.output": "Output" } diff --git a/src/MeshWeaver.Testing.InMesh/MeshTestCase.cs b/src/MeshWeaver.Testing.InMesh/MeshTestCase.cs new file mode 100644 index 0000000000..690cda8365 --- /dev/null +++ b/src/MeshWeaver.Testing.InMesh/MeshTestCase.cs @@ -0,0 +1,50 @@ +using System; +using System.Reactive; + +namespace MeshWeaver.Testing.InMesh; + +/// +/// One explicitly listed case of a Tests area — the shape for a suite whose cases are static +/// methods rather than classes. Rendered through +/// , +/// which runs the cases one after another, streams each one's progress and output, and bounds each +/// by its (or the area's deadline). +/// +/// Two kinds, because they fail to finish in two different ways. A SYNCHRONOUS case +/// () is a method that asserts and returns; it runs as one leaf on +/// the mesh's Tests I/O pool, so a body that never returns costs its own bound and a pool +/// slot — never the render thread, which is what made one hung case freeze the whole page. A LIVE +/// case () returns a cold observable +/// that emits once when the assertion held; the bound disposes its subscription, which is what +/// cancels the work behind it. +/// +public sealed record MeshTestCase +{ + /// What the row says. + public required string Name { get; init; } + + /// This case's own bound; null uses the area's deadline. + public TimeSpan? Timeout { get; init; } + + /// The synchronous body, handed a line writer for its output. Null for a live case. + public Action>? Synchronous { get; init; } + + /// The live body, handed a line writer for its output. Null for a synchronous case. + public Func, IObservable>? Reactive { get; init; } + + /// A synchronous case: passes when returns, fails with its exception's message. + public static MeshTestCase Of(string name, Action body, TimeSpan? timeout = null) => + new() { Name = name, Timeout = timeout, Synchronous = _ => body() }; + + /// A synchronous case that writes output lines while it runs. + public static MeshTestCase Of(string name, Action> body, TimeSpan? timeout = null) => + new() { Name = name, Timeout = timeout, Synchronous = body }; + + /// A live case: passes on the body's first emission; an error, an empty completion or the bound fails it. + public static MeshTestCase Live(string name, Func> body, TimeSpan? timeout = null) => + new() { Name = name, Timeout = timeout, Reactive = _ => body() }; + + /// A live case that writes output lines while it runs. + public static MeshTestCase Live(string name, Func, IObservable> body, TimeSpan? timeout = null) => + new() { Name = name, Timeout = timeout, Reactive = body }; +} diff --git a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs new file mode 100644 index 0000000000..35c3a3e89b --- /dev/null +++ b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs @@ -0,0 +1,323 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Reactive; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Layout.DataGrid; +using MeshWeaver.Mesh.Threading; +using Microsoft.Extensions.DependencyInjection; + +namespace MeshWeaver.Testing.InMesh; + +/// +/// The STREAMING half of the runner: what a Tests area shows while its cases run. +/// +/// 🚨 A Tests area used to emit ONE frame, at the end. A case that called an external service +/// live, or one that hung, therefore left the page on "Rendering …" for its whole budget — with no +/// way to tell a slow case from a stuck one, and nothing on screen to say which case it was. Now the +/// area renders every case as pending the moment it opens, then the running case with its elapsed +/// time (ticking) and every output line it has written so far, and each finished case with its +/// verdict. Only the LAST frame is the verdict. +/// +/// 🚨 Progress frames carry so a consumer that +/// classifies the first frame it can (the plugin gate) keeps waiting for the verdict. And they are +/// written so that even a consumer that predates the id cannot mistake one for a verdict: a case +/// that has passed SO FAR shows ✔, one that failed ✖, and the title counts cases "done" — never +/// the ✅ / ❌ glyphs or the "N/M passed" sentence a verdict carries. +/// +public static partial class MeshTestRunner +{ + /// What a pending row's result column shows. + public const string PendingResult = "⏳"; + + /// What the running row's result column shows. + public const string RunningResult = "▶"; + + /// The prefix of every verdict a case earns by exceeding its bound. + public const string TimedOutPrefix = "timed out: "; + + /// How often a progress frame is emitted while a case runs (its elapsed time ticks). + public static readonly TimeSpan DefaultProgressInterval = TimeSpan.FromSeconds(1); + + /// One frame of a run: every case's current row, and whether this is the verdict. + /// Every case of the run, in run order — pending, running or finished. + /// True on the one final snapshot, after the last case finished. + /// Time since the run started. + public sealed record TestRunSnapshot(IReadOnlyList Cases, bool Complete, TimeSpan Elapsed) + { + /// Cases that have a verdict (passed, failed or skipped). + public int Done => Cases.Count(c => !IsUnfinished(c)); + } + + /// A row of the rendered table (the data the grid binds). + /// The test class (or suite, for listed cases). + /// The case's display name. + /// The status glyph and word. + /// Elapsed, in seconds. + /// The failure message and the case's output lines. + public sealed record CaseRow(string Class, string Case, string Result, string Time, string Output); + + /// The column titles, resolved ONCE in the viewer's language while the render scope still carries it. + /// Title of the class column. + /// Title of the case column. + /// Title of the result column. + /// Title of the time column. + /// Title of the output column. + /// The progress title format: suite, done, total, seconds. + public sealed record ColumnTitles(string Class, string Case, string Result, string Time, string Output, string ProgressTitle) + { + /// The English titles, for a render with no viewer. + public static readonly ColumnTitles English = new("Class", "Case", "Result", "Time", "Output", "{0} tests — {1} of {2} done, running for {3}s"); + + /// The titles in the viewer's language. + public static ColumnTitles For(LayoutAreaHost host) => new( + host.Localize("tests.column.class"), + host.Localize("tests.column.case"), + host.Localize("tests.column.result"), + host.Localize("tests.column.time"), + host.Localize("tests.column.output"), + host.Localize("tests.progress", "{0}", "{1}", "{2}", "{3}")); + } + + /// + /// Runs the classes' cases one after another and emits a snapshot of EVERY case: first all + /// pending, then — at most once per — the running case's elapsed + /// time and output so far, and finally the complete verdict (). + /// + /// The area host; null runs the classes that need no mesh. + /// The test classes. + /// The per-case bound when a case declares none. + /// The pool the cases run on; null resolves the mesh's Tests pool. + /// How often a running case's row is refreshed; null = . + public static IObservable Progress(LayoutAreaHost? host, IEnumerable classes, TimeSpan deadline, IIoPool? pool = null, TimeSpan? interval = null) + { + var list = classes.ToList(); + var plan = list.SelectMany(cls => Cases(cls).Select(c => (cls.Name, c.Name))).ToImmutableArray(); + return Track(plan, (started, line) => Run(host, list, deadline, pool, started, line), interval ?? DefaultProgressInterval); + } + + /// + /// Runs listed cases one after another and emits snapshots exactly as the class-based overload does. + /// + /// The area host; null runs without a mesh (synchronous cases on an unbounded pool). + /// What the class column says. + /// The cases, in run order. + /// The per-case bound when a case declares none. + /// The pool synchronous cases run on; null resolves the mesh's Tests pool. + /// How often a running case's row is refreshed; null = . + public static IObservable Progress(LayoutAreaHost? host, string suite, IReadOnlyList cases, TimeSpan deadline, IIoPool? pool = null, TimeSpan? interval = null) + { + var plan = cases.Select(c => (suite, c.Name)).ToImmutableArray(); + var resolved = pool ?? host?.Hub.ServiceProvider.GetService()?.Get(IoPoolNames.Tests) ?? IoPool.Unbounded; + return Track(plan, + (started, line) => cases.Select(c => RunListed(suite, c, deadline, resolved, started, line)).Concat(), + interval ?? DefaultProgressInterval); + } + + // One listed case: its clock, its output, its bound — and every outcome, including "completed + // without ever emitting", turned into a row rather than into silence. + private static IObservable RunListed(string suite, MeshTestCase c, TimeSpan deadline, IIoPool pool, Action onStarted, Action onLine) => + Observable.Defer(() => + { + var bound = c.Timeout ?? deadline; + var started = DateTimeOffset.UtcNow; + var output = new OutputLines(onLine); + onStarted(); + var work = c.Synchronous is { } body + ? pool.InvokeBlocking(_ => { body(output.Add); return Unit.Default; }) + : Observable.Defer(() => c.Reactive is { } live + ? live(output.Add) + : Observable.Throw(new InvalidOperationException($"the case '{c.Name}' has no body"))); + CaseResult Verdict(string? failure) => failure is null + ? new CaseResult(suite, c.Name, "✅ pass", output.Joined, DateTimeOffset.UtcNow - started) + : new CaseResult(suite, c.Name, "❌ FAIL", failure + (output.Joined.Length > 0 ? " · " + output.Joined : ""), DateTimeOffset.UtcNow - started); + return work + .Select(_ => (string?)null) + .Take(1) + .DefaultIfEmpty("the case completed without an outcome") + .Timeout(bound, Observable.Return($"{TimedOutPrefix}no verdict within {bound.TotalSeconds:F0}s")) + .Catch(ex => Observable.Return(Unwrap(ex).Message)) + .Select(Verdict); + }); + + // A case's output lines: kept for its verdict, and forwarded to the progress frame as written. + // Lines may arrive from any thread the case happens to run on, hence the interlocked swap. + private sealed class OutputLines(Action forward) + { + private ImmutableList lines = ImmutableList.Empty; + + public void Add(string line) + { + ImmutableInterlocked.Update(ref lines, l => l.Add(line)); + forward(line); + } + + public string Joined => string.Join(" · ", lines); + } + + // ———————————————————————————————————————————————————————— the snapshot fold + + private abstract record RunEvent; + private sealed record StartedEvent : RunEvent; + private sealed record LineEvent(string Text) : RunEvent; + private sealed record FinishedEvent(CaseResult Result) : RunEvent; + private sealed record TickEvent : RunEvent; + private sealed record DoneEvent : RunEvent; + + private sealed record RunState( + ImmutableArray Rows, + int Running, + DateTimeOffset RunningSince, + ImmutableList RunningOutput, + DateTimeOffset StartedAt, + bool Dirty, + bool Emit, + bool Complete); + + private static IObservable Track( + ImmutableArray<(string Class, string Name)> plan, + Func, IObservable> run, + TimeSpan interval) => + Observable.Defer(() => + { + // Started/line signals come from inside the cases (any thread), finished verdicts from the + // run itself; Merge serialises them into ONE ordered fold, so no state here is shared. + var signals = Subject.Synchronize(new Subject()); + var now = DateTimeOffset.UtcNow; + var initial = new RunState( + [.. plan.Select(p => new CaseResult(p.Class, p.Name, PendingResult, "", TimeSpan.Zero))], + -1, now, [], now, Dirty: false, Emit: true, Complete: false); + var finished = run(() => signals.OnNext(new StartedEvent()), line => signals.OnNext(new LineEvent(line))) + .Select(result => (RunEvent)new FinishedEvent(result)); + return finished + .Publish(verdicts => + { + var over = verdicts.LastOrDefaultAsync(); + return Observable.Merge( + verdicts, + signals.TakeUntil(over), + Observable.Interval(interval).Select(_ => (RunEvent)new TickEvent()).TakeUntil(over)); + }) + .Concat(Observable.Return(new DoneEvent())) + .Scan(initial, Apply) + .StartWith(initial) + .Where(state => state.Emit) + .Select(state => new TestRunSnapshot(state.Rows, state.Complete, DateTimeOffset.UtcNow - state.StartedAt)); + }); + + // Started, line and verdict only mark the state DIRTY; a tick (or the end) is what emits, so a + // suite of fifty instant cases renders a handful of frames, not a hundred. + private static RunState Apply(RunState state, RunEvent e) + { + var at = DateTimeOffset.UtcNow; + switch (e) + { + case StartedEvent: + { + var index = FirstUnfinished(state.Rows); + if (index < 0) return state with { Emit = false }; + var row = state.Rows[index] with { Result = RunningResult, Detail = "", Elapsed = TimeSpan.Zero }; + return state with { Rows = state.Rows.SetItem(index, row), Running = index, RunningSince = at, RunningOutput = [], Dirty = true, Emit = false }; + } + case LineEvent line when state.Running >= 0: + { + var output = state.RunningOutput.Add(line.Text); + var row = state.Rows[state.Running] with { Detail = string.Join(" · ", output) }; + return state with { Rows = state.Rows.SetItem(state.Running, row), RunningOutput = output, Dirty = true, Emit = false }; + } + case FinishedEvent done: + { + var index = state.Running >= 0 ? state.Running : FirstUnfinished(state.Rows); + if (index < 0) return state with { Emit = false }; + return state with { Rows = state.Rows.SetItem(index, done.Result), Running = -1, RunningOutput = [], Dirty = true, Emit = false }; + } + case TickEvent: + { + if (state.Running < 0) + return state with { Emit = state.Dirty, Dirty = false }; + var row = state.Rows[state.Running] with { Elapsed = at - state.RunningSince }; + return state with { Rows = state.Rows.SetItem(state.Running, row), Emit = true, Dirty = false }; + } + case DoneEvent: + return state with { Complete = true, Emit = true, Dirty = false }; + default: + return state with { Emit = false }; + } + } + + private static int FirstUnfinished(ImmutableArray rows) + { + for (var i = 0; i < rows.Length; i++) + if (IsUnfinished(rows[i])) return i; + return -1; + } + + private static bool IsUnfinished(CaseResult row) => + row.Result is PendingResult or RunningResult; + + // ———————————————————————————————————————————————————————— rendering + + private static IObservable Frames(LayoutAreaHost host, string suite, IObservable snapshots) + { + // Resolved HERE, in the render scope that carries the viewer — the frames below are built on + // timer and pool threads, where no AccessContext is set and every key would fall to English. + var titles = ColumnTitles.For(host); + return snapshots.Select(snapshot => (UiControl?)(snapshot.Complete + ? Render(suite, snapshot.Cases, titles) + : RenderProgress(suite, snapshot, titles))); + } + + /// + /// The verdict frame — the gate's contract: a title carrying "N/M passed" and a ✅/❌ table. + /// + /// The suite name. + /// Every case's verdict. + /// The column titles. + public static UiControl Render(string suite, IReadOnlyList results, ColumnTitles titles) => + Controls.Stack.WithWidth("100%") + .WithView(Controls.Title(Summary(suite, results), 2), "Title") + .WithView(Grid(results.Select(r => Row(r, r.Result, r.Detail)), titles), "Cases"); + + /// + /// A progress frame: , a title that counts the + /// cases DONE (never "passed"), and ✔ / ✖ for a finished case — the verdict glyphs are reserved + /// for the verdict frame. + /// + /// The suite name. + /// The run so far. + /// The column titles and progress title. + public static UiControl RenderProgress(string suite, TestRunSnapshot snapshot, ColumnTitles titles) => + Controls.Stack.WithWidth("100%") + .WithView(Controls.Title(string.Format(System.Globalization.CultureInfo.InvariantCulture, titles.ProgressTitle, + suite, snapshot.Done, snapshot.Cases.Count, (int)snapshot.Elapsed.TotalSeconds), 2), "Title") + .WithView(Controls.Progress("", snapshot.Cases.Count == 0 ? 100 : snapshot.Done * 100 / snapshot.Cases.Count), "Progress") + .WithView(Grid(snapshot.Cases.Select(r => Row(r, ProgressResult(r.Result), Neutral(r.Detail))), titles), "Cases") + .WithId(AreaFrameClassifier.TestsRunningId); + + private static CaseRow Row(CaseResult r, string result, string detail) => + new(r.Class, r.Name, result, IsUnfinished(r) && r.Result == PendingResult ? "" : $"{r.Elapsed.TotalSeconds:0.0}s", detail); + + private static DataGridControl Grid(IEnumerable rows, ColumnTitles titles) => + Controls.DataGrid(rows.ToImmutableArray()) + .WithColumn( + new PropertyColumnControl { Property = "class" }.WithTitle(titles.Class), + new PropertyColumnControl { Property = "case" }.WithTitle(titles.Case), + new PropertyColumnControl { Property = "result" }.WithTitle(titles.Result), + new PropertyColumnControl { Property = "time" }.WithTitle(titles.Time).WithAlign("end"), + new PropertyColumnControl { Property = "output" }.WithTitle(titles.Output)); + + // A finished case in a PROGRESS frame: passed-so-far / failed-so-far, never the verdict glyphs. + private static string ProgressResult(string result) => + result.StartsWith("✅", StringComparison.Ordinal) ? "✔" + : result.StartsWith("❌", StringComparison.Ordinal) ? "✖" + : result; + + // A case's own output may carry the verdict glyphs too (a nested report, a copied line); in a + // progress frame they are neutralised for the same reason as the result column. + private static string Neutral(string text) => + text.Replace("✅", "✔", StringComparison.Ordinal).Replace("❌", "✖", StringComparison.Ordinal); +} diff --git a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs index 85407d416e..2b369da99f 100644 --- a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs +++ b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs @@ -22,8 +22,12 @@ namespace MeshWeaver.Testing.InMesh; /// Cases run one after another (the mesh is shared; a class gets its own partition); each is /// bounded by a deadline and a failure carries the exception's message. Nothing here needs setup: /// the mesh the area renders in is the fixture. +/// +/// The area STREAMS: it renders every case as pending the moment it is opened, then each +/// case as running (with its elapsed time and its output lines as they arrive) and finished, and +/// only the last frame carries the verdict — see . /// -public static class MeshTestRunner +public static partial class MeshTestRunner { /// The default per-case deadline. public static readonly TimeSpan DefaultDeadline = TimeSpan.FromSeconds(30); @@ -48,11 +52,24 @@ public static IReadOnlyList TestClasses(Assembly assembly) => public static IObservable Area(LayoutAreaHost host, string suite, Assembly assembly, TimeSpan? deadline = null) => Area(host, suite, TestClasses(assembly), deadline); - /// The Tests area over the given classes. + /// + /// The Tests area over the given classes: a progress frame per second while the cases + /// run, then the verdict frame (see ). + /// public static IObservable Area(LayoutAreaHost host, string suite, IEnumerable classes, TimeSpan? deadline = null) => - Run(host, classes, deadline ?? DefaultDeadline) - .ToList() - .Select(results => (UiControl?)Render(suite, results.ToList())); + Frames(host, suite, Progress(host, classes, deadline ?? DefaultDeadline)); + + /// + /// The Tests area over explicitly listed cases — the shape of a Tests area whose cases + /// are static methods rather than classes. Streams exactly as + /// the class-based area does. + /// + /// The area host. + /// The suite name the verdict title carries. + /// The cases, run one after another in this order. + /// The per-case bound when a case declares none. + public static IObservable Area(LayoutAreaHost host, string suite, IReadOnlyList cases, TimeSpan? deadline = null) => + Frames(host, suite, Progress(host, suite, cases, deadline ?? DefaultDeadline)); /// Executes the cases, one at a time, emitting each verdict as it lands. /// A null host runs the classes that need no mesh (parameterless constructors) — the runner's own tests use it. @@ -62,22 +79,31 @@ public static IReadOnlyList TestClasses(Assembly assembly) => /// The pool the cases run on. Null resolves the mesh's /// pool (or without a host); the runner's own tests pass a bounded one. public static IObservable Run(LayoutAreaHost? host, IEnumerable classes, TimeSpan deadline, IIoPool? pool = null) => + Run(host, classes, deadline, pool, NoSignal, NoLine); + + private static void NoSignal() { } + + private static void NoLine(string _) { } + + private static IObservable Run(LayoutAreaHost? host, IEnumerable classes, TimeSpan deadline, IIoPool? pool, Action onStarted, Action onLine) => Observable.Defer(() => { // The cases of THIS run that ignored their cancellation and are therefore still holding a // pool slot. Per run, never static: two Tests areas rendering at once do not share it. var leaked = new List(); - return classes.Select(cls => RunClass(host, cls, deadline, pool, leaked)).Concat(); + return classes.Select(cls => RunClass(host, cls, deadline, pool, leaked, onStarted, onLine)).Concat(); }); - private static IObservable RunClass(LayoutAreaHost? host, Type cls, TimeSpan deadline, IIoPool? requestedPool, List leaked) + private static IObservable RunClass(LayoutAreaHost? host, Type cls, TimeSpan deadline, IIoPool? requestedPool, List leaked, Action onStarted, Action onLine) { var partition = $"{MeshTestContext.TestRoot}/{cls.Name}-{Guid.NewGuid():N}"[..Math.Min(80, MeshTestContext.TestRoot.Length + 1 + cls.Name.Length + 33)]; var cases = Cases(cls).ToList(); return Observable.Defer(() => { var output = new List(); - var context = host is null ? null : new MeshTestContext(host, partition, output.Add, deadline); + // Every line a case writes lands in its verdict's detail AND streams to the progress + // frame the moment it is written — a slow case shows what it is doing while it does it. + var context = host is null ? null : new MeshTestContext(host, partition, line => { output.Add(line); onLine(line); }, deadline); object? instance; MeshTestContext.Current = context; try @@ -94,7 +120,7 @@ private static IObservable RunClass(LayoutAreaHost? host, Type cls, // the leaf's token to the subscription, which is what lets the bound below CANCEL a case // rather than abandon it. Host-less runs (the runner's own tests) have no registry. var pool = requestedPool ?? host?.Hub.ServiceProvider.GetService()?.Get(IoPoolNames.Tests) ?? IoPool.Unbounded; - return cases.Select(c => RunCase(instance, cls, c, output, deadline, context, pool, leaked)).Concat(); + return cases.Select(c => RunCase(instance, cls, c, output, deadline, context, pool, leaked, onStarted)).Concat(); }); } @@ -107,7 +133,12 @@ private static IObservable RunClass(LayoutAreaHost? host, Type cls, /// public static readonly TimeSpan CancellationGrace = TimeSpan.FromSeconds(2); - private static IObservable RunCase(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked) + private static IObservable RunCase(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked, Action onStarted) => + // Deferred so the clock, the leak check and the "running" signal all belong to the moment + // the case actually STARTS — not to the moment Concat asked for the observable. + Observable.Defer(() => RunCaseNow(instance, cls, c, output, deadline, context, pool, leaked, onStarted)); + + private static IObservable RunCaseNow(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked, Action onStarted) { if (c.Skip is not null) return Observable.Return(new CaseResult(cls.Name, c.Name, "⏭ skipped", c.Skip, TimeSpan.Zero)); @@ -121,6 +152,7 @@ private static IObservable RunCase(object? instance, Type cls, TestC var bound = c.TimeoutSeconds > 0 ? TimeSpan.FromSeconds(c.TimeoutSeconds) : deadline; var started = DateTimeOffset.UtcNow; output.Clear(); + onStarted(); // The leaf signals its own unwinding. When the bound elapses, Timeout disposes the leaf's // subscription, the pool cancels the token it handed the case, and the runner waits the grace // on this signal: a case that observed the token completes it; one that did not leaves it @@ -154,9 +186,9 @@ private static IObservable RunCase(object? instance, Type cls, TestC .Select(_ => new CaseResult(cls.Name, c.Name, "✅ pass", string.Join(" · ", output), DateTimeOffset.UtcNow - started)) .Catch(_ => unwound .Timeout(CancellationGrace) - .Select(_ => Fail($"no verdict within {bound.TotalSeconds:F0}s — cancelled and unwound")) + .Select(_ => Fail($"{TimedOutPrefix}no verdict within {bound.TotalSeconds:F0}s — cancelled and unwound")) .Catch(_ => Observable.Return(Leak(leaked, $"{cls.Name}.{c.Name}", Fail( - $"no verdict within {bound.TotalSeconds:F0}s — and the case IGNORED its cancellation token: still running {CancellationGrace.TotalSeconds:F0}s after it was cancelled. Pass MeshTestContext.CancellationToken (or a trailing CancellationToken parameter) into what the case awaits"))))) + $"{TimedOutPrefix}no verdict within {bound.TotalSeconds:F0}s — and the case IGNORED its cancellation token: still running {CancellationGrace.TotalSeconds:F0}s after it was cancelled. Pass MeshTestContext.CancellationToken (or a trailing CancellationToken parameter) into what the case awaits"))))) .Catch(ex => Observable.Return(Fail(Unwrap(ex).Message))); } @@ -224,13 +256,9 @@ public static string Table(IReadOnlyList results) => "| Class | Case | Result | Time | Detail |\n|---|---|---|---:|---|\n" + string.Join("\n", results.Select(r => $"| {r.Class} | {Escape(r.Name)} | {r.Result} | {r.Elapsed.TotalSeconds:0.0}s | {Escape(r.Detail)} |")); - /// The gate's contract: a title carrying "N/M passed" and a ✅/❌ table. - public static UiControl Render(string suite, IReadOnlyList results) - { - return Controls.Stack.WithWidth("100%") - .WithView(Controls.Title(Summary(suite, results), 2), "Title") - .WithView(Controls.Markdown(Table(results)), "Cases"); - } + /// The gate's contract: a title carrying "N/M passed" and a ✅/❌ table (English column titles). + public static UiControl Render(string suite, IReadOnlyList results) => + Render(suite, results, ColumnTitles.English); private static string Escape(string s) => s.Replace("|", "\\|").Replace("\n", " "); } diff --git a/test/MeshWeaver.Cli.Test/TestsCommandTest.cs b/test/MeshWeaver.Cli.Test/TestsCommandTest.cs new file mode 100644 index 0000000000..5b67a59b6c --- /dev/null +++ b/test/MeshWeaver.Cli.Test/TestsCommandTest.cs @@ -0,0 +1,66 @@ +using MeshWeaver.Cli; +using Xunit; + +namespace MeshWeaver.Cli.Test; + +/// +/// memex tests reads a node's Tests area frame by frame. Pinned over the PURE seam: which +/// frame is progress and which is the verdict, what a verdict says, and that a row is printed again +/// only when its state or output changed — never for a ticking clock. +/// +public class TestsCommandTest +{ + private const string Progress = """ + {"areas":{ + "\"$Menu:Node\"":{"items":[{"label":"Request approval","icon":"✅"}]}, + "\"Tests\"":{"$type":"StackControl","id":"tests-running","areas":[{"area":"Tests/Title"},{"area":"Tests/Cases"}]}, + "\"Tests/Title\"":{"$type":"HtmlControl","data":"

Store tests — 1 of 3 done, running for 4s

"}, + "\"Tests/Cases\"":{"$type":"DataGridControl","data":[ + {"class":"Store","case":"first","result":"✔","time":"0.1s","output":""}, + {"class":"Store","case":"slow","result":"▶","time":"3.9s","output":"contacted the service"}, + {"class":"Store","case":"last","result":"⏳","time":"","output":""}]}}} + """; + + private const string Verdict = """ + {"areas":{ + "\"Tests\"":{"$type":"StackControl","areas":[{"area":"Tests/Title"},{"area":"Tests/Cases"}]}, + "\"Tests/Title\"":{"$type":"HtmlControl","data":"

Store tests — 2/3 passed

"}, + "\"Tests/Cases\"":{"$type":"DataGridControl","data":[ + {"class":"Store","case":"first","result":"✅ pass","time":"0.1s","output":""}, + {"class":"Store","case":"slow","result":"❌ FAIL","time":"45.0s","output":"timed out: no verdict within 45s · contacted the service"}, + {"class":"Store","case":"last","result":"✅ pass","time":"0.0s","output":""}]}}} + """; + + [Fact] + public void A_progress_frame_is_running_and_names_every_case() + { + var frame = TestsCommand.Parse(Progress); + Assert.True(frame.Running); + Assert.Equal("Store tests — 1 of 3 done, running for 4s", frame.Title); + Assert.Equal(["first", "slow", "last"], frame.Rows.Select(r => r.Case)); + Assert.Equal("contacted the service", frame.Rows[1].Output); + } + + [Fact] + public void The_verdict_frame_fails_on_a_failed_case_and_the_chrome_never_counts() + { + var frame = TestsCommand.Parse(Verdict); + Assert.False(frame.Running); + Assert.False(frame.Passed); + + var green = TestsCommand.Parse(Verdict.Replace("❌ FAIL", "✅ pass").Replace("2/3", "3/3")); + Assert.True(green.Passed); + } + + [Fact] + public void A_row_is_reprinted_only_when_its_state_or_output_changes() + { + var printed = new Dictionary(); + Assert.Equal(3, TestsCommand.Changes(printed, TestsCommand.Parse(Progress)).Count); + Assert.Empty(TestsCommand.Changes(printed, TestsCommand.Parse(Progress.Replace("3.9s", "4.9s")))); + + var verdictLines = TestsCommand.Changes(printed, TestsCommand.Parse(Verdict)); + Assert.Equal(3, verdictLines.Count); + Assert.Contains(verdictLines, l => l.Contains("timed out", StringComparison.Ordinal)); + } +} diff --git a/test/MeshWeaver.PluginTester.Test/AreaProbeTest.cs b/test/MeshWeaver.PluginTester.Test/AreaProbeTest.cs index b6e8b22cb0..d1e86ddb1e 100644 --- a/test/MeshWeaver.PluginTester.Test/AreaProbeTest.cs +++ b/test/MeshWeaver.PluginTester.Test/AreaProbeTest.cs @@ -85,6 +85,32 @@ public IDisposable Subscribe(IObserver observer) } } + // A Tests area's PROGRESS frame (MeshTestRunner streams them while its cases run). Its rows may + // carry anything — here even a verdict glyph — and it must still never classify: the id says the + // verdict frame follows. + private static readonly JsonElement ProgressFrame = Frame( + """{"areas":{"Tests":{"id":"tests-running","title":"Store tests — 1 of 2 done, running for 3s","rows":["✔ First_Passes","❌ looks like a verdict","▶ Second_Runs"]}}}"""); + + /// + /// A streamed progress frame is TRANSIENT: the verdict comes from the frame that follows it, and a + /// run cut off mid-way says how far it got. + /// + [Fact] + public async Task ProgressFrame_ThenGreenTable_IsPassed_AndACutRunNamesItsProgress() + { + var verdict = await AreaProbe.ClassifyTestsFrames(FrameStream(ProgressFrame, GreenTable), TimeSpan.FromSeconds(5)) + .FirstAsync().Await(); + Assert.Equal(CheckOutcome.Passed, verdict.Outcome); + Assert.Equal("2/2 passed", verdict.Detail); + + var cut = await AreaProbe.ClassifyTestsFrames( + Observable.Return(ProgressFrame).Concat(Observable.Never()), TimeSpan.FromMilliseconds(300)) + .FirstAsync().Await(); + Assert.Equal(CheckOutcome.Failed, cut.Outcome); + Assert.Contains("still running its cases", cut.Detail); + Assert.Contains("1 of 2 done", cut.Detail); + } + /// /// The regression pin: a not-found frame followed by the real table must be GREEN. Before the /// fix, Take(1) latched the not-found frame and the run failed without the tests ever running. diff --git a/test/MeshWeaver.Testing.InMesh.Test/MeshTestRunnerTests.cs b/test/MeshWeaver.Testing.InMesh.Test/MeshTestRunnerTests.cs index 50b945b84b..5172e2bbfa 100644 --- a/test/MeshWeaver.Testing.InMesh.Test/MeshTestRunnerTests.cs +++ b/test/MeshWeaver.Testing.InMesh.Test/MeshTestRunnerTests.cs @@ -2,6 +2,7 @@ using System.Threading; using System; using System.Linq; +using System.Reactive; using System.Reactive.Linq; using System.Threading.Tasks; using MeshWeaver.Testing.InMesh; @@ -116,6 +117,66 @@ public async Task A_pool_filled_by_leaked_cases_is_named_not_timed_out() Assert.Equal(TimeSpan.Zero, blocked.Elapsed); } + /// + /// The area STREAMS: all cases pending at once, the running case with its output while it runs, + /// a hung synchronous case failed by its bound instead of freezing the run, and exactly ONE + /// verdict snapshot — the last. The page that sat on "Rendering …" for a whole live suite is the + /// defect this pins. + /// + [Fact] + public async Task Progress_streams_pending_running_output_and_one_verdict_last() + { + var cases = new[] + { + MeshTestCase.Live("slow", log => + { + log("contacted the service"); + return Observable.Timer(TimeSpan.FromMilliseconds(400)).Select(_ => Unit.Default); + }), + MeshTestCase.Of("hangs", () => Thread.Sleep(TimeSpan.FromSeconds(1)), timeout: TimeSpan.FromMilliseconds(150)), + MeshTestCase.Of("fails", () => throw new InvalidOperationException("the assertion message")), + MeshTestCase.Live("never answers", () => Observable.Empty()), + }; + + var snapshots = await MeshTestRunner + .Progress(null, "Streams", cases, TestTimeouts.Quick, interval: TimeSpan.FromMilliseconds(50)) + .ToList().Await(); + + var first = snapshots[0]; + Assert.False(first.Complete); + Assert.All(first.Cases, c => Assert.Equal(MeshTestRunner.PendingResult, c.Result)); + Assert.Equal(4, first.Cases.Count); + + Assert.Contains(snapshots, s => !s.Complete + && s.Cases[0].Result == MeshTestRunner.RunningResult + && s.Cases[0].Detail.Contains("contacted the service", StringComparison.Ordinal)); + + Assert.Single(snapshots, s => s.Complete); + var verdict = snapshots[^1]; + Assert.True(verdict.Complete, "the verdict is the LAST snapshot"); + Assert.True(verdict.Cases[0].Passed, verdict.Cases[0].Detail); + Assert.Contains("contacted the service", verdict.Cases[0].Detail); + Assert.StartsWith("❌", verdict.Cases[1].Result); + Assert.StartsWith(MeshTestRunner.TimedOutPrefix, verdict.Cases[1].Detail); + Assert.Contains("the assertion message", verdict.Cases[2].Detail); + Assert.Contains("completed without an outcome", verdict.Cases[3].Detail); + + // A progress frame is transient and can never be read as a verdict, even by a consumer that + // ignores the id: no ✅ / ❌ and no "N/M passed" anywhere in it. + var midRun = snapshots.First(s => !s.Complete && s.Done > 0); + var progress = MeshTestRunner.RenderProgress("Streams", midRun, MeshTestRunner.ColumnTitles.English); + Assert.True(MeshWeaver.Layout.AreaFrameClassifier.IsTestsRunning(progress)); + Assert.True(MeshWeaver.Layout.AreaFrameClassifier.IsTransientFrame(progress)); + var progressJson = System.Text.Json.JsonSerializer.Serialize(progress); + Assert.DoesNotContain("✅", progressJson); + Assert.DoesNotContain("❌", progressJson); + Assert.DoesNotMatch(@"\d+\s*/\s*\d+\s+passed", progressJson); + + var final = MeshTestRunner.Render("Streams", verdict.Cases, MeshTestRunner.ColumnTitles.English); + Assert.False(MeshWeaver.Layout.AreaFrameClassifier.IsTransientFrame(final)); + Assert.Equal("Streams tests — 1/4 passed", MeshTestRunner.Summary("Streams", verdict.Cases)); + } + [Fact] public void Discovers_only_classes_with_cases() { diff --git a/tools/MeshWeaver.PluginTester/AreaProbe.cs b/tools/MeshWeaver.PluginTester/AreaProbe.cs index 2078833fdf..c5ba82c679 100644 --- a/tools/MeshWeaver.PluginTester/AreaProbe.cs +++ b/tools/MeshWeaver.PluginTester/AreaProbe.cs @@ -37,6 +37,10 @@ public static class AreaProbe // one can use the same constant the typed predicates use — never on the localized prose. private static readonly string CompileProgressMarker = AreaFrameClassifier.CompileProgressId; + // A Tests area's PROGRESS frame (MeshTestRunner streams one per second while its cases run) — + // transient in the same way: the verdict frame follows. Matched on the control Id, never prose. + private static readonly string TestsRunningMarker = AreaFrameClassifier.TestsRunningId; + private static readonly Regex PassSummary = new( @"(\d+)\s*/\s*(\d+)\s+passed", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -168,6 +172,17 @@ private static IObservable Frames(IMessageHub client, string nodePa return null; } + // 🚨 A Tests area STREAMS its progress (MeshTestRunner): every frame before the verdict + // carries this id, and its rows already show cases that passed or failed so far. Classifying + // one would green a suite whose later cases have not run — so it is a promise, like the + // compile page, and the timeout verdict reports how far the run got. + if (strings.Any(s => string.Equals(s, TestsRunningMarker, StringComparison.Ordinal))) + { + onTransient("the Tests area was still running its cases — last progress: " + + (strings.FirstOrDefault(s => s.Contains(" — ", StringComparison.Ordinal)) ?? "(no title)")); + return null; + } + var renderError = strings.FirstOrDefault(s => s.Contains(RenderFailedMarker, StringComparison.Ordinal) || s.Contains(RenderEmergencyMarker, StringComparison.Ordinal)); From 069e63fdd30c7342a2485e3e3a0b5afc38c1f160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:21:56 +0200 Subject: [PATCH 2/5] feat(testing): run_tests as an activity; review fixes (line attribution, sync-body leaks, i18n, culture) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MeshOperations.RunTests(path): holds ONE subscription to the node's Tests area and writes it to an activity in the caller's partition (category TestRun) — a line per case, its output, and a terminal status that is the verdict. Polling get @node/area/Tests re-runs the suite on every read (measured on memex-cloud: each read filed the Maintenance suite's live request nodes again and answered with an earlier subscription's cached verdict). - TestsAreaFrame (MeshWeaver.Layout): the pure reader of a Tests area frame, keyed by class+case. - memex tests: POST api/mesh/run-tests, then polls the ACTIVITY (starts nothing); every request bounded by --timeout; immutable state. - Runner: a case's writer is closed at its verdict, and the class-wide writer admits only the running case's lines (AsyncLocal token), so a case that outlived its bound cannot write onto the next row; a timed-out synchronous listed body is named as still holding its pool slot; verdict words localized beside the glyphs; elapsed formatted invariant. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/MeshWeaver.Cli/MemexClient.cs | 2 + src/MeshWeaver.Cli/TestsCommand.cs | 225 ++++++------------ .../ActivityCategory.cs | 6 + .../Data/Architecture/WritingTests.md | 25 +- src/MeshWeaver.Layout/TestsAreaFrame.cs | 160 +++++++++++++ .../MeshOperations.RunTests.cs | 156 ++++++++++++ .../MeshOperations.cs | 2 +- .../MeshWeaver.Mesh.Operations.csproj | 3 + .../Localization/strings.de.json | 8 +- .../Localization/strings.en.json | 8 +- .../MeshTestRunner.Progress.cs | 62 ++++- .../MeshTestRunner.cs | 55 ++++- .../Memex.Portal.Shared.Test.csproj | 3 + .../RunTestsStreamsIntoAnActivityTest.cs | 73 ++++++ test/MeshWeaver.Cli.Test/TestsCommandTest.cs | 70 +++--- .../TestsAreaFrameTest.cs | 55 +++++ 16 files changed, 690 insertions(+), 223 deletions(-) create mode 100644 src/MeshWeaver.Layout/TestsAreaFrame.cs create mode 100644 src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs create mode 100644 test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs create mode 100644 test/MeshWeaver.Layout.Test/TestsAreaFrameTest.cs diff --git a/src/MeshWeaver.Cli/MemexClient.cs b/src/MeshWeaver.Cli/MemexClient.cs index 25d26518e1..3e1559afd4 100644 --- a/src/MeshWeaver.Cli/MemexClient.cs +++ b/src/MeshWeaver.Cli/MemexClient.cs @@ -53,6 +53,8 @@ public Task Recycle(string path, string? reason, CancellationToken ct) = public Task Compile(string path, CancellationToken ct) => Post("api/mesh/compile", new { path }, ct); public Task Diagnostics(string path, CancellationToken ct) => Post("api/mesh/diagnostics", new { path }, ct); public Task ExecuteScript(string path, int timeoutSeconds, CancellationToken ct) => Post("api/mesh/execute-script", new { path, timeoutSeconds }, ct); + /// Runs a node's Tests area as an activity (MeshOperations.RunTests); answers {status, activityPath}. + public Task RunTests(string path, int timeoutSeconds, CancellationToken ct) => Post("api/mesh/run-tests", new { path, timeoutSeconds }, ct); public Task NavigateTo(string path, CancellationToken ct) => Post("api/mesh/navigate-to", new { path }, ct); public Task BaseUrl(CancellationToken ct) => Post("api/mesh/base-url", new { }, ct); diff --git a/src/MeshWeaver.Cli/TestsCommand.cs b/src/MeshWeaver.Cli/TestsCommand.cs index 6a157448dc..0a905bf2ad 100644 --- a/src/MeshWeaver.Cli/TestsCommand.cs +++ b/src/MeshWeaver.Cli/TestsCommand.cs @@ -1,188 +1,113 @@ +using System.Collections.Immutable; using System.Text.Json; -using System.Text.RegularExpressions; namespace MeshWeaver.Cli; /// -/// memex tests <path> — run (= render) one node's Tests area and print its -/// progress as a console, line by line, until the verdict frame arrives. +/// memex tests <path> — run one node's Tests area and print its progress as a +/// console until the verdict. /// -/// The Tests area IS the runner (MeshTestRunner): rendering it executes the cases, and -/// the area streams a progress frame per second — every case pending, running (with its elapsed -/// time and its output so far) or finished — then ONE verdict frame. This command reads the area -/// through the same get @path/area/Tests verb an agent uses, prints every row whose state or -/// output changed since the last read, and exits on the verdict: 0 when every counted case -/// passed, 1 when any failed, 4 when no verdict arrived within --timeout (the -/// last progress is printed, so a hung case is NAMED rather than waited on). +/// 🚨 It does NOT poll get @path/area/Tests. Rendering the area RUNS the suite, and every +/// one-shot read opens a fresh subscription — measured on memex.meshweaver.cloud, each read filed the +/// Maintenance suite's live request nodes again, and answered with whatever verdict an earlier +/// subscription had cached. Instead the portal runs the area ONCE, holding one subscription, as an +/// activity (POST api/mesh/run-tests → MeshOperations.RunTests) that logs a line per +/// case as it starts, writes output and lands; this command polls that ACTIVITY node, which starts +/// nothing, and prints each new line. /// -/// A Tests area that predates the streaming runner renders only its verdict, so the first read -/// simply prints the finished table — the command works against every portal, streaming or not. +/// Exit codes: 0 the activity Succeeded (every counted case passed), 1 it Failed +/// (a case failed, the area has none, or the portal's own bound elapsed — its last line names the +/// case still running), 4 no terminal status within --timeout. /// public static class TestsCommand { - /// The UiControl.Id every progress frame carries (AreaFrameClassifier.TestsRunningId). - public const string TestsRunningId = "tests-running"; - - private static readonly Regex PassSummary = new(@"(\d+)\s*/\s*(\d+)\s+passed", RegexOptions.CultureInvariant); - - /// One case as the area renders it. - /// The case's name. - /// Its status glyph / verdict. - /// Its elapsed time. - /// Its failure message and output lines. - public sealed record Row(string Case, string Result, string Time, string Output); - - /// What one read of the area says. - /// True for a progress frame — the verdict is still to come. - /// The frame's title (progress or verdict), when it has one. - /// The cases the frame lists as grid rows (empty for a legacy markdown table). - /// Every other string the Tests area carries (a legacy table's markdown). - public sealed record Frame(bool Running, string? Title, IReadOnlyList Rows, IReadOnlyList Text) + /// One read of the activity node. + /// The activity status (Running, Succeeded, Failed, …). + /// The true number of lines ever written. + /// The most recent lines (older ones are archived under {activity}/_Log). + public sealed record ActivityRead(string Status, int MessageCount, ImmutableArray Window) { - /// True when the verdict frame says every counted case passed. - public bool Passed - { - get - { - var all = Rows.Select(r => r.Result).Concat(Text).Append(Title ?? ""); - if (all.Any(s => s.Contains('❌'))) - return false; - var summary = all.Select(s => PassSummary.Match(s)).FirstOrDefault(m => m.Success); - return summary is null - ? Rows.Count > 0 || all.Any(s => s.Contains('✅')) - : summary.Groups[1].Value == summary.Groups[2].Value; - } - } + /// True once the run has a terminal status. + public bool Terminal => Status is not ("Running" or "Pending" or ""); } - /// - /// Reads a get @path/area/Tests answer. Pure over the JSON: the chrome areas the framework - /// writes into every subscription ($Menu…, $Banner) are skipped, exactly as the - /// plugin gate skips them. - /// - /// The rendered area store. - public static Frame Parse(string json) + /// Reads a get @activity answer. Pure over the JSON. + /// The activity node. + public static ActivityRead ParseActivity(string json) { using var doc = JsonDocument.Parse(json); - var rows = new List(); - var text = new List(); - string? title = null; - var running = false; - if (doc.RootElement.ValueKind == JsonValueKind.Object - && doc.RootElement.TryGetProperty("areas", out var areas) - && areas.ValueKind == JsonValueKind.Object) - { - foreach (var area in areas.EnumerateObject()) - { - if (area.Name.TrimStart('"').StartsWith('$')) - continue; - Walk(area.Value); - } - } - return new Frame(running, title, rows, text); - - void Walk(JsonElement e) - { - switch (e.ValueKind) - { - case JsonValueKind.Object: - if (e.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String - && id.GetString() == TestsRunningId) - running = true; - if (e.TryGetProperty("case", out var c) && e.TryGetProperty("result", out var r)) - { - rows.Add(new Row(Str(c), Str(r), Str(e, "time"), Str(e, "output"))); - return; - } - foreach (var p in e.EnumerateObject()) - Walk(p.Value); - break; - case JsonValueKind.Array: - foreach (var item in e.EnumerateArray()) - Walk(item); - break; - case JsonValueKind.String: - var s = e.GetString() ?? ""; - if (title is null && (s.Contains(" tests — ", StringComparison.Ordinal) || s.Contains("-Tests — ", StringComparison.Ordinal))) - title = StripTags(s); - else if (s.Contains('✅') || s.Contains('❌') || PassSummary.IsMatch(s)) - text.Add(s); - break; - } - } + var content = doc.RootElement.TryGetProperty("content", out var c) ? c : doc.RootElement; + var status = content.TryGetProperty("status", out var s) && s.ValueKind == JsonValueKind.String ? s.GetString() ?? "" : ""; + var messages = content.TryGetProperty("messages", out var m) && m.ValueKind == JsonValueKind.Array + ? m.EnumerateArray() + .Select(e => e.TryGetProperty("message", out var text) ? text.GetString() ?? "" : "") + .ToImmutableArray() + : []; + var count = content.TryGetProperty("messageCount", out var n) && n.TryGetInt32(out var value) ? value : messages.Length; + return new ActivityRead(status, count, messages); } - private static string Str(JsonElement e) => e.ValueKind == JsonValueKind.String ? e.GetString() ?? "" : e.ToString(); - - private static string Str(JsonElement e, string property) => - e.TryGetProperty(property, out var v) ? Str(v) : ""; - - private static string StripTags(string s) => Regex.Replace(s, "<[^>]+>", "").Trim(); - /// - /// The lines to print for given what was already printed: a row that is - /// new, or whose result or output changed. A row whose only change is its ticking time is not - /// reprinted — the console would scroll with nothing new in it. + /// The lines of not yet printed, given that + /// lines were printed before — plus a note when some slid out of the window unseen. /// - /// The rows as last printed, by case name. - /// The frame just read. - public static IReadOnlyList Changes(IDictionary printed, Frame next) + /// How many lines have been printed so far. + /// The activity just read. + public static ImmutableArray NewLines(int printed, ActivityRead read) { - var lines = new List(); - foreach (var row in next.Rows) - { - if (printed.TryGetValue(row.Case, out var before) - && before.Result == row.Result && before.Output == row.Output) - continue; - printed[row.Case] = row; - lines.Add($"{row.Result,-10} {row.Case}{(row.Time.Length > 0 ? $" ({row.Time})" : "")}{(row.Output.Length > 0 ? $" — {row.Output}" : "")}"); - } - return lines; + var fresh = read.MessageCount - printed; + if (fresh <= 0) + return []; + var shown = Math.Min(fresh, read.Window.Length); + var lines = read.Window.Skip(read.Window.Length - shown); + return fresh > shown + ? [$"… {fresh - shown} line(s) were archived before they could be read (see the activity's _Log)", .. lines] + : [.. lines]; } - /// Reads the area until the verdict, printing each change; returns the exit code. + /// Starts the run, then prints the activity's lines until it is terminal; returns the exit code. /// The portal client. /// The node whose Tests area to run. - /// How long to wait for the verdict. - /// How often to read the area. + /// How long to wait for a terminal status. + /// How often to read the activity. /// Where the console goes. /// Cancels the wait. public static async Task Run(MemexClient client, string path, TimeSpan timeout, TimeSpan interval, TextWriter output, CancellationToken ct) { - var area = $"@{path.TrimStart('@').TrimEnd('/')}/area/Tests"; - var printed = new Dictionary(StringComparer.Ordinal); - var deadline = DateTimeOffset.UtcNow + timeout; - string? lastTitle = null; - while (true) + // The command's --timeout bounds EVERY request too, so a hung read cannot outlive it. + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct); + deadline.CancelAfter(timeout); + try { - var body = await client.Get(area, ct); - if (body.StartsWith("Error:", StringComparison.Ordinal) || body.StartsWith("Not found", StringComparison.Ordinal)) + var started = await client.RunTests(path, (int)Math.Ceiling(timeout.TotalSeconds), deadline.Token); + using var dispatched = JsonDocument.Parse(started); + if (dispatched.RootElement.TryGetProperty("status", out var s) && s.GetString() != "Dispatched" + || !dispatched.RootElement.TryGetProperty("activityPath", out var activity)) { - await output.WriteLineAsync(body); + await output.WriteLineAsync(started); return 1; } - var frame = Parse(body); - if (frame.Title is { } title && title != lastTitle) - { - await output.WriteLineAsync($"== {title}"); - lastTitle = title; - } - foreach (var line in Changes(printed, frame)) - await output.WriteLineAsync(line); - if (!frame.Running) + var activityPath = activity.GetString()!; + await output.WriteLineAsync($"== {activityPath}"); + var printed = 0; + while (true) { - foreach (var line in frame.Text) + var read = ParseActivity(await client.Get("@" + activityPath, deadline.Token)); + foreach (var line in NewLines(printed, read)) await output.WriteLineAsync(line); - return frame.Passed ? 0 : 1; + printed = Math.Max(printed, read.MessageCount); + if (read.Terminal) + { + await output.WriteLineAsync($"== {read.Status}"); + return read.Status == "Succeeded" ? 0 : 1; + } + await Task.Delay(interval, deadline.Token); } - if (DateTimeOffset.UtcNow >= deadline) - { - var stuck = printed.Values.Where(r => r.Result == "▶").Select(r => r.Case).ToList(); - await output.WriteLineAsync( - $"no verdict within {timeout.TotalSeconds:F0}s — still running: {(stuck.Count > 0 ? string.Join(", ", stuck) : "(no case reported running)")}"); - return 4; - } - await Task.Delay(interval, ct); + } + catch (OperationCanceledException) when (deadline.IsCancellationRequested && !ct.IsCancellationRequested) + { + await output.WriteLineAsync($"no terminal status within {timeout.TotalSeconds:F0}s"); + return 4; } } } diff --git a/src/MeshWeaver.Data.Contract/ActivityCategory.cs b/src/MeshWeaver.Data.Contract/ActivityCategory.cs index a74f9d7ea5..4d9c517124 100644 --- a/src/MeshWeaver.Data.Contract/ActivityCategory.cs +++ b/src/MeshWeaver.Data.Contract/ActivityCategory.cs @@ -27,6 +27,12 @@ public static class ActivityCategory /// public const string WriteConflict = nameof(WriteConflict); + /// + /// Activity carrying a run of a node's Tests area: one line per case as it starts, + /// writes output and reaches its verdict (run_tests / memex tests). + /// + public const string TestRun = nameof(TestRun); + /// /// Activity whose category is unknown or unclassified. /// diff --git a/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md b/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md index a78482872d..2dc6cd6674 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md @@ -170,16 +170,21 @@ case from a stuck one and nothing naming the case. It now streams: Static methods: `MeshTestRunner.Area(host, suite, cases)` over `MeshTestCase.Of(name, method)` (synchronous) and `MeshTestCase.Live(name, () => observable, timeout)` (hosted, passes on first emission); either overload hands the body a line writer whose output streams into the row. -- **From a terminal or an agent:** `memex tests @` reads the area every two seconds, prints each - case as its state or output changes, and exits `0` (all passed), `1` (a failure) or `4` (no verdict - within `--timeout`, naming the case still running). An agent does the same with - `get @/area/Tests`, which answers with the frame the area has reached. 🚨 Rendering RUNS the - suite, so what a second read shows depends on whether it reuses the first read's subscription: - measured once on memex.meshweaver.cloud, a read of `Admin/Maintenance/…/Tests` answered the - finished verdict in 0.9 s for a suite whose live cases alone take 18 s or more — i.e. it reused a - subscription that had already run. A read that opens a NEW subscription (another replica, an - evicted stream) renders the area anew and so runs the suite — live cases included — again; that - half is inferred from the render path, not measured. Treat each fresh subscription as a fresh run. +- **From a terminal or an agent — run it as an ACTIVITY, never by polling the area.** 🚨 Rendering + the area RUNS the suite, and every one-shot `get @/area/Tests` opens a fresh subscription. + Measured on memex.meshweaver.cloud (`Admin/Maintenance/refresh-app-tiles-20260828-mainnode-3`): two + `get` reads 34 s apart each filed a new pair of the Maintenance suite's live request nodes + (`Admin/Maintenance/mnt-…`, 10:51:00Z and 10:51:34Z), and the second read answered in 0.9 s with the + verdict an EARLIER subscription had cached while its own run had only just begun. So a poll of the + area re-runs every live case and still does not show the run it started. + `MeshOperations.RunTests(path)` holds ONE subscription for the whole run and writes it to an + activity in the caller's own partition (category `TestRun`): a line per case as it runs and as it + lands, with its output; `Succeeded` when every counted case passed, `Failed` otherwise, or when no + verdict arrived within the bound (the last line names the case still running). It answers at once + with `{status: "Dispatched", activityPath}`; poll `get @{activityPath}`, which starts nothing. + `memex tests @` is that loop in a terminal (`POST api/mesh/run-tests`, then the activity every + two seconds), exiting `0` (passed), `1` (failed) or `4` (no terminal status within `--timeout`). The + portal route and the MCP `run_tests` tool live with the portal in MeshWeaver.Plugins. ## The Canonical Test Base diff --git a/src/MeshWeaver.Layout/TestsAreaFrame.cs b/src/MeshWeaver.Layout/TestsAreaFrame.cs new file mode 100644 index 0000000000..731ef9acd9 --- /dev/null +++ b/src/MeshWeaver.Layout/TestsAreaFrame.cs @@ -0,0 +1,160 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.RegularExpressions; +using MeshWeaver.Data; + +namespace MeshWeaver.Layout; + +/// +/// One read of a Tests area as it travels the sync stream (a serialized area store) — the +/// shape a consumer that is NOT the page needs: is the run still going, which cases are where, and +/// what did the verdict say. Pure over the JSON. +/// +/// The framework writes chrome into every area subscription ($Menu…, $Banner, +/// $Dialog); chrome never counts — the Approvals menu entry's icon IS the ✅ emoji. +/// +/// False until the requested area's own control has landed. +/// True for a frame that a later one replaces: a streamed progress frame +/// () or the compile-progress page. +/// True when the hub answered that no renderer exists for the area. +/// The frame's title line (progress or verdict), when it has one. +/// The cases the frame lists as grid rows (empty for a table rendered as markdown). +/// Every other content string that carries a verdict glyph or summary. +public sealed record TestsAreaFrame( + bool Materialized, + bool Transient, + bool NotFound, + string? Title, + ImmutableArray Rows, + ImmutableArray Text) +{ + /// One case as the area renders it. + /// The class (or suite) column. + /// The case's name. + /// Its status glyph / verdict. + /// Its elapsed time. + /// Its failure message and output lines. + public sealed record Row(string Class, string Case, string Result, string Time, string Output) + { + /// The key a case is tracked by across frames — class AND case, because two + /// classes can carry a case of the same name. + public string Key => $"{Class}\u001f{Case}"; + } + + private static readonly Regex PassSummary = new(@"(\d+)\s*/\s*(\d+)\s+passed", RegexOptions.CultureInvariant); + + /// True when the frame is a VERDICT and every counted case passed. + public bool Passed + { + get + { + if (!Materialized || Transient || NotFound) + return false; + var all = Rows.Select(r => r.Result).Concat(Text).Append(Title ?? "").ToImmutableArray(); + if (all.Any(s => s.Contains('❌'))) + return false; + var summary = all.Select(s => PassSummary.Match(s)).FirstOrDefault(m => m.Success); + return summary is null + ? all.Any(s => s.Contains('✅')) + : summary.Groups[1].Value == summary.Groups[2].Value; + } + } + + /// Reads one serialized area store for the area . + /// The frame as the sync stream carries it. + /// The area name (normally Tests). + public static TestsAreaFrame Read(JsonElement store, string area = "Tests") + { + var rows = ImmutableArray.CreateBuilder(); + var text = ImmutableArray.CreateBuilder(); + string? title = null; + var transient = false; + var notFound = false; + var materialized = false; + if (store.ValueKind == JsonValueKind.Object + && store.TryGetProperty(LayoutAreaReference.Areas, out var areas) + && areas.ValueKind == JsonValueKind.Object) + { + foreach (var entry in areas.EnumerateObject()) + { + var name = entry.Name.Trim('"'); + if (name.StartsWith('$')) + continue; + if (name == area && entry.Value.ValueKind is not (JsonValueKind.Null or JsonValueKind.Undefined)) + materialized = true; + if (name == area || name.StartsWith(area + "/", StringComparison.Ordinal)) + Walk(entry.Value); + } + } + return new TestsAreaFrame(materialized, transient, notFound, title, rows.ToImmutable(), text.ToImmutable()); + + void Walk(JsonElement e) + { + switch (e.ValueKind) + { + case JsonValueKind.Object: + if (e.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) + { + var value = id.GetString(); + if (value is AreaFrameClassifier.TestsRunningId or AreaFrameClassifier.CompileProgressId) + transient = true; + else if (value == AreaFrameClassifier.AreaNotFoundId) + notFound = true; + } + if (e.TryGetProperty("case", out var c) && e.TryGetProperty("result", out var r)) + { + rows.Add(new Row(Str(e, "class"), Str(c), Str(r), Str(e, "time"), Str(e, "output"))); + return; + } + foreach (var p in e.EnumerateObject()) + Walk(p.Value); + break; + case JsonValueKind.Array: + foreach (var item in e.EnumerateArray()) + Walk(item); + break; + case JsonValueKind.String: + var s = e.GetString() ?? ""; + if (s.Contains("**Area not found**", StringComparison.Ordinal)) + notFound = true; + else if (title is null && s.Contains(" — ", StringComparison.Ordinal) && s.Contains("ests", StringComparison.Ordinal)) + title = Regex.Replace(s, "<[^>]+>", "").Trim(); + else if (s.Contains('✅') || s.Contains('❌') || PassSummary.IsMatch(s)) + text.Add(s); + break; + } + } + } + + private static string Str(JsonElement e) => e.ValueKind == JsonValueKind.String ? e.GetString() ?? "" : e.ToString(); + + private static string Str(JsonElement e, string property) => + e.TryGetProperty(property, out var v) ? Str(v) : ""; + + /// + /// The rows of that are new, or whose result or output changed, since + /// — and the state to carry to the next frame. A row whose only + /// change is its ticking time is not a change. + /// + /// The rows as last reported, by . + /// The frame just read. + public static (ImmutableArray Changed, ImmutableDictionary Printed) Changes( + ImmutableDictionary printed, TestsAreaFrame next) + { + var changed = ImmutableArray.CreateBuilder(); + foreach (var row in next.Rows) + { + if (printed.TryGetValue(row.Key, out var before) + && before.Result == row.Result && before.Output == row.Output) + continue; + printed = printed.SetItem(row.Key, row); + changed.Add(row); + } + return (changed.ToImmutable(), printed); + } + + /// One console line for a row. + /// The row. + public static string Line(Row row) => + $"{row.Result} {row.Case}{(row.Time.Length > 0 ? $" ({row.Time})" : "")}{(row.Output.Length > 0 ? $" — {row.Output}" : "")}"; +} diff --git a/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs b/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs new file mode 100644 index 0000000000..b8bc312bde --- /dev/null +++ b/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs @@ -0,0 +1,156 @@ +using System.Collections.Immutable; +using System.Reactive; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Data; +using MeshWeaver.GitSync; +using MeshWeaver.Layout; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +public partial class MeshOperations +{ + /// The default bound on a whole run. + public const int DefaultRunTestsSeconds = 600; + + /// + /// Runs a node's Tests area as an ACTIVITY and returns as soon as the activity exists: + /// {status:'Dispatched', activityPath}. The activity (in the CALLER's own partition) + /// gets one line per case as it starts, writes output and reaches its verdict, and ends + /// Succeeded when every counted case passed, Failed otherwise — including when no + /// verdict arrived within , which names the case still running. + /// + /// 🚨 Why an activity and not a poll of get @node/area/Tests: rendering the area RUNS + /// the suite, and a one-shot read opens a fresh subscription each time — measured on + /// memex.meshweaver.cloud, two get reads 34 s apart each filed a new pair of the + /// Maintenance suite's live request nodes, and the second answered with an OLD verdict frame + /// while its own run had only just started. This keeps ONE subscription for the life of the run and turns its frames into a + /// log anyone can poll without starting anything. + /// + /// The node whose Tests area to run. + /// The bound on the whole run. + public IObservable RunTests(string path, int timeoutSeconds = DefaultRunTestsSeconds) + { + logger.LogInformation("RunTests called with path={Path}", path); + if (string.IsNullOrWhiteSpace(path)) + return Observable.Return(Json(new { status = "Error", message = "path is required" })); + var resolvedPath = ResolvePath(path).Trim('/'); + var userId = ResolveCallerUserId(); + if (userId == WellKnownUsers.Anonymous) + return Observable.Return(Json(new + { + status = "Error", + path = resolvedPath, + message = "run_tests needs a signed-in caller: the run's activity is written to the caller's own partition.", + })); + var budget = TimeSpan.FromSeconds(Math.Clamp(timeoutSeconds, 1, 3600)); + var accessService = hub.ServiceProvider.GetService(); + var pathResolver = hub.ServiceProvider.GetRequiredService(); + + return Observable.Create(observer => + { + var answered = 0; + void Answer(string json) + { + if (Interlocked.Exchange(ref answered, 1) != 0) return; + observer.OnNext(json); + observer.OnCompleted(); + } + var caller = accessService?.Context ?? accessService?.CircuitContext; + // The run OUTLIVES this call on purpose: the caller gets the activity path and polls it. + // Its lifetime is the activity's — bounded by the budget below, cancellable through the + // activity's own RequestedStatus, and tracked by ActivityRunner for teardown. + hub.RunActivity(userId, ActivityCategory.TestRun, + new LogMessage($"Run the Tests area of {resolvedPath}", LogLevel.Information) + .WithKey("activity.tests.title", ("path", (object?)resolvedPath)), + ctx => WatchTestsArea(resolvedPath, caller, pathResolver, budget, ctx), + activityPath => Answer(Json(new { status = "Dispatched", path = resolvedPath, activityPath }))) + .Subscribe( + _ => { }, + ex => + { + logger.LogWarning(ex, "RunTests could not run the Tests area of {Path}", resolvedPath); + Answer(Json(new { status = "Error", path = resolvedPath, message = ex.Message })); + }); + return Disposable.Empty; + }); + } + + private string Json(object value) => JsonSerializer.Serialize(value, hub.JsonSerializerOptions); + + // ONE subscription to the Tests area for the whole run: every changed row becomes a log line, + // and the first frame that is not a progress frame is the verdict. + private IObservable WatchTestsArea( + string nodePath, AccessContext? caller, IPathResolver pathResolver, TimeSpan budget, ActivityContext ctx) => + Observable.Defer(() => + { + // The last frame seen, so a timeout can NAME the case that never finished. Written only + // from the frame pipeline, which the stream delivers serially. + TestsAreaFrame? last = null; + return WatchTestsFrames(nodePath, caller, pathResolver, ctx) + .Do(frame => last = frame) + .SkipWhile(frame => frame.Transient) + .Take(1) + .Timeout(budget) + .Catch(_ => + { + var running = last?.Rows.Where(r => r.Result == "▶").Select(r => r.Case).ToImmutableArray() ?? []; + return Observable.Throw(new TimeoutException( + $"no verdict from the Tests area of {nodePath} within {budget.TotalSeconds:F0}s — " + + (running.Length > 0 ? $"still running: {string.Join(", ", running)}" : "no case reported running"))); + }) + .Select(frame => + { + if (frame.NotFound) + throw new InvalidOperationException($"{nodePath} has no Tests area."); + foreach (var line in frame.Text) + ctx.Log(new LogMessage(line, LogLevel.Information)); + if (!frame.Passed) + throw new InvalidOperationException( + $"{frame.Title ?? nodePath + " tests"}: not every case passed."); + return Unit.Default; + }); + }); + + private IObservable WatchTestsFrames( + string nodePath, AccessContext? caller, IPathResolver pathResolver, ActivityContext ctx) => + pathResolver.ResolvePath(nodePath) + .Take(1) + .SelectMany(resolution => resolution is null + ? Observable.Throw(new InvalidOperationException($"Not found: {nodePath}")) + : Observable.Defer(() => + { + var accessService = hub.ServiceProvider.GetService(); + // Same identity rule as RenderArea: the subscribe must carry the CALLER, or the + // owner's read gate would be bypassed under System. + using var scope = caller is not null ? accessService?.SwitchAccessContext(caller) : null; + var stream = hub.StreamSubscribingHub().GetWorkspace() + .GetRemoteStream( + (Address)resolution.Prefix, new LayoutAreaReference("Tests") { Id = "" }); + return stream.Select(change => TestsAreaFrame.Read(change.Value)) + .Finally(stream.Dispose); + })) + .Where(frame => frame.Materialized) + .Scan((Frame: (TestsAreaFrame?)null, Printed: ImmutableDictionary.Empty), + (state, frame) => + { + var (changed, printed) = TestsAreaFrame.Changes(state.Printed, frame); + foreach (var row in changed) + ctx.Log(new LogMessage(TestsAreaFrame.Line(row), + row.Result.StartsWith('❌') || row.Result.StartsWith('✖') ? LogLevel.Warning : LogLevel.Information)); + return (frame, printed); + }) + .Select(state => state.Frame!) + .Do(frame => + { + if (!frame.Transient && frame.Title is { } title) + ctx.Log(new LogMessage(title, LogLevel.Information)); + }); +} diff --git a/src/MeshWeaver.Mesh.Operations/MeshOperations.cs b/src/MeshWeaver.Mesh.Operations/MeshOperations.cs index 72a73903d9..2fbc9d0430 100644 --- a/src/MeshWeaver.Mesh.Operations/MeshOperations.cs +++ b/src/MeshWeaver.Mesh.Operations/MeshOperations.cs @@ -48,7 +48,7 @@ namespace MeshWeaver.AI; /// bridge at an external boundary (.FirstAsync().ToTask()) — never inside hub /// flow. See CLAUDE.md "NOTHING ASYNC EVER". /// -public class MeshOperations +public partial class MeshOperations { private readonly IMessageHub hub; private readonly ILogger logger; diff --git a/src/MeshWeaver.Mesh.Operations/MeshWeaver.Mesh.Operations.csproj b/src/MeshWeaver.Mesh.Operations/MeshWeaver.Mesh.Operations.csproj index ec320e15ab..40400d75df 100644 --- a/src/MeshWeaver.Mesh.Operations/MeshWeaver.Mesh.Operations.csproj +++ b/src/MeshWeaver.Mesh.Operations/MeshWeaver.Mesh.Operations.csproj @@ -8,6 +8,9 @@ + + diff --git a/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json b/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json index 88f7482efc..8825efb778 100644 --- a/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json +++ b/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json @@ -1717,10 +1717,14 @@ "ui.threadQueueStateRetried": "Neu gestartet", "ui.threadQueueStateActionFailed": "Nicht ausführbar", "ui.threadQueueStateFailed": "Fehlgeschlagen", - "tests.progress": "{0}-Tests — {1} von {2} erledigt, seit {3}s in Arbeit", + "activity.tests.title": "Tests von {path} ausführen", + "tests.progress":"{0}-Tests — {1} von {2} erledigt, seit {3}s in Arbeit", "tests.column.class": "Klasse", "tests.column.case": "Fall", "tests.column.result": "Ergebnis", "tests.column.time": "Zeit", - "tests.column.output": "Ausgabe" + "tests.column.output": "Ausgabe", + "tests.result.passed": "bestanden", + "tests.result.failed": "FEHLER", + "tests.result.skipped": "übersprungen" } diff --git a/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json b/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json index 946953d853..4c59338ebe 100644 --- a/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json +++ b/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json @@ -1717,10 +1717,14 @@ "ui.threadQueueStateRetried": "Relaunched", "ui.threadQueueStateActionFailed": "Could not act", "ui.threadQueueStateFailed": "Failed", - "tests.progress": "{0} tests — {1} of {2} done, running for {3}s", + "activity.tests.title": "Run the Tests area of {path}", + "tests.progress":"{0} tests — {1} of {2} done, running for {3}s", "tests.column.class": "Class", "tests.column.case": "Case", "tests.column.result": "Result", "tests.column.time": "Time", - "tests.column.output": "Output" + "tests.column.output": "Output", + "tests.result.passed": "pass", + "tests.result.failed": "FAIL", + "tests.result.skipped": "skipped" } diff --git a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs index 35c3a3e89b..31a2da86da 100644 --- a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs +++ b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.Progress.cs @@ -68,10 +68,15 @@ public sealed record CaseRow(string Class, string Case, string Result, string Ti /// Title of the time column. /// Title of the output column. /// The progress title format: suite, done, total, seconds. - public sealed record ColumnTitles(string Class, string Case, string Result, string Time, string Output, string ProgressTitle) + /// The word beside ✅ in a verdict row. + /// The word beside ❌ in a verdict row. + /// The word beside ⏭ in a verdict row. + public sealed record ColumnTitles(string Class, string Case, string Result, string Time, string Output, string ProgressTitle, + string Passed, string Failed, string Skipped) { /// The English titles, for a render with no viewer. - public static readonly ColumnTitles English = new("Class", "Case", "Result", "Time", "Output", "{0} tests — {1} of {2} done, running for {3}s"); + public static readonly ColumnTitles English = new("Class", "Case", "Result", "Time", "Output", + "{0} tests — {1} of {2} done, running for {3}s", "pass", "FAIL", "skipped"); /// The titles in the viewer's language. public static ColumnTitles For(LayoutAreaHost host) => new( @@ -80,7 +85,17 @@ public sealed record ColumnTitles(string Class, string Case, string Result, stri host.Localize("tests.column.result"), host.Localize("tests.column.time"), host.Localize("tests.column.output"), - host.Localize("tests.progress", "{0}", "{1}", "{2}", "{3}")); + host.Localize("tests.progress", "{0}", "{1}", "{2}", "{3}"), + host.Localize("tests.result.passed"), + host.Localize("tests.result.failed"), + host.Localize("tests.result.skipped")); + + // The verdict word in the viewer's language; the GLYPH is the contract every reader matches. + internal string Display(string result) => + result.StartsWith("✅", StringComparison.Ordinal) ? $"✅ {Passed}" + : result.StartsWith("❌", StringComparison.Ordinal) ? $"❌ {Failed}" + : result.StartsWith("⏭", StringComparison.Ordinal) ? $"⏭ {Skipped}" + : result; } /// @@ -126,36 +141,61 @@ private static IObservable RunListed(string suite, MeshTestCase c, T var bound = c.Timeout ?? deadline; var started = DateTimeOffset.UtcNow; var output = new OutputLines(onLine); + var bodyReturned = 0; onStarted(); var work = c.Synchronous is { } body - ? pool.InvokeBlocking(_ => { body(output.Add); return Unit.Default; }) + ? pool.InvokeBlocking(_ => + { + try { body(output.Add); } + finally { Volatile.Write(ref bodyReturned, 1); } + return Unit.Default; + }) : Observable.Defer(() => c.Reactive is { } live ? live(output.Add) : Observable.Throw(new InvalidOperationException($"the case '{c.Name}' has no body"))); - CaseResult Verdict(string? failure) => failure is null - ? new CaseResult(suite, c.Name, "✅ pass", output.Joined, DateTimeOffset.UtcNow - started) - : new CaseResult(suite, c.Name, "❌ FAIL", failure + (output.Joined.Length > 0 ? " · " + output.Joined : ""), DateTimeOffset.UtcNow - started); + // 🚨 A synchronous body takes no token, so the bound can fail it but not STOP it: it keeps + // its Tests-pool slot until it returns. Say so in the verdict instead of letting a later + // case's "not run"/timeout be the first anyone hears of it. + string TimedOut() => + $"{TimedOutPrefix}no verdict within {bound.TotalSeconds:F0}s" + + (c.Synchronous is not null && Volatile.Read(ref bodyReturned) == 0 + ? " — the synchronous body is still running and holds a Tests-pool slot until it returns" + : ""); + CaseResult Verdict(string? failure) + { + // Terminal: whatever the body writes from here on belongs to no row. + output.Close(); + return failure is null + ? new CaseResult(suite, c.Name, "✅ pass", output.Joined, DateTimeOffset.UtcNow - started) + : new CaseResult(suite, c.Name, "❌ FAIL", failure + (output.Joined.Length > 0 ? " · " + output.Joined : ""), DateTimeOffset.UtcNow - started); + } return work .Select(_ => (string?)null) .Take(1) .DefaultIfEmpty("the case completed without an outcome") - .Timeout(bound, Observable.Return($"{TimedOutPrefix}no verdict within {bound.TotalSeconds:F0}s")) + .Timeout(bound, Observable.Defer(() => Observable.Return(TimedOut()))) .Catch(ex => Observable.Return(Unwrap(ex).Message)) .Select(Verdict); }); // A case's output lines: kept for its verdict, and forwarded to the progress frame as written. // Lines may arrive from any thread the case happens to run on, hence the interlocked swap. + // Once the case has its verdict the writer is CLOSED: a body that outlived its bound and keeps + // writing must not land its lines on the next case's row. private sealed class OutputLines(Action forward) { private ImmutableList lines = ImmutableList.Empty; + private int closed; public void Add(string line) { + if (Volatile.Read(ref closed) != 0) return; ImmutableInterlocked.Update(ref lines, l => l.Add(line)); forward(line); } + public void Close() => Volatile.Write(ref closed, 1); + public string Joined => string.Join(" · ", lines); } @@ -280,7 +320,7 @@ private static bool IsUnfinished(CaseResult row) => public static UiControl Render(string suite, IReadOnlyList results, ColumnTitles titles) => Controls.Stack.WithWidth("100%") .WithView(Controls.Title(Summary(suite, results), 2), "Title") - .WithView(Grid(results.Select(r => Row(r, r.Result, r.Detail)), titles), "Cases"); + .WithView(Grid(results.Select(r => Row(r, titles.Display(r.Result), r.Detail)), titles), "Cases"); /// /// A progress frame: , a title that counts the @@ -299,7 +339,9 @@ public static UiControl RenderProgress(string suite, TestRunSnapshot snapshot, C .WithId(AreaFrameClassifier.TestsRunningId); private static CaseRow Row(CaseResult r, string result, string detail) => - new(r.Class, r.Name, result, IsUnfinished(r) && r.Result == PendingResult ? "" : $"{r.Elapsed.TotalSeconds:0.0}s", detail); + new(r.Class, r.Name, result, + r.Result == PendingResult ? "" : r.Elapsed.TotalSeconds.ToString("0.0", System.Globalization.CultureInfo.InvariantCulture) + "s", + detail); private static DataGridControl Grid(IEnumerable rows, ColumnTitles titles) => Controls.DataGrid(rows.ToImmutableArray()) diff --git a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs index 2b369da99f..fbc168130d 100644 --- a/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs +++ b/src/MeshWeaver.Testing.InMesh/MeshTestRunner.cs @@ -101,9 +101,19 @@ private static IObservable RunClass(LayoutAreaHost? host, Type cls, return Observable.Defer(() => { var output = new List(); + var gate = new CaseGate(); // Every line a case writes lands in its verdict's detail AND streams to the progress // frame the moment it is written — a slow case shows what it is doing while it does it. - var context = host is null ? null : new MeshTestContext(host, partition, line => { output.Add(line); onLine(line); }, deadline); + // 🚨 Only the RUNNING case's lines: the writer is shared by the class, and a case that + // outlived its bound can still write after the next case started. Its lines carry its own + // token (the AsyncLocal flows into its continuations) and are dropped, never pinned on the + // next case's row. + var context = host is null ? null : new MeshTestContext(host, partition, line => + { + if (!gate.Admits()) return; + output.Add(line); + onLine(line); + }, deadline); object? instance; MeshTestContext.Current = context; try @@ -120,10 +130,39 @@ private static IObservable RunClass(LayoutAreaHost? host, Type cls, // the leaf's token to the subscription, which is what lets the bound below CANCEL a case // rather than abandon it. Host-less runs (the runner's own tests) have no registry. var pool = requestedPool ?? host?.Hub.ServiceProvider.GetService()?.Get(IoPoolNames.Tests) ?? IoPool.Unbounded; - return cases.Select(c => RunCase(instance, cls, c, output, deadline, context, pool, leaked, onStarted)).Concat(); + return cases.Select(c => RunCase(instance, cls, c, output, deadline, context, pool, leaked, onStarted, gate)).Concat(); }); } + // Which case's writes the class-wide writer currently admits. A case's continuations carry its + // token through the ExecutionContext; a line with a token that is not the running case's is a + // late write from a case that already has its verdict. A line with NO token (a thread that did + // not flow the context) is attributed to the running case — the best the writer can know. + private sealed class CaseGate + { + private static readonly AsyncLocal caseToken = new(); + private object? running; + + public object Open() + { + var token = new object(); + Volatile.Write(ref running, token); + return token; + } + + public static void Enter(object token) => caseToken.Value = token; + + public void Close(object token) => Interlocked.CompareExchange(ref running, null, token); + + public bool Admits() + { + var current = Volatile.Read(ref running); + if (current is null) return false; + var mine = caseToken.Value; + return mine is null || ReferenceEquals(mine, current); + } + } + /// /// How long the runner waits, after a case's bound elapsed and its token was cancelled, for the /// case to unwind before reporting that it IGNORED the token. A case that observes @@ -133,12 +172,17 @@ private static IObservable RunClass(LayoutAreaHost? host, Type cls, /// public static readonly TimeSpan CancellationGrace = TimeSpan.FromSeconds(2); - private static IObservable RunCase(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked, Action onStarted) => + private static IObservable RunCase(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked, Action onStarted, CaseGate gate) => // Deferred so the clock, the leak check and the "running" signal all belong to the moment // the case actually STARTS — not to the moment Concat asked for the observable. - Observable.Defer(() => RunCaseNow(instance, cls, c, output, deadline, context, pool, leaked, onStarted)); + Observable.Defer(() => + { + var token = gate.Open(); + return RunCaseNow(instance, cls, c, output, deadline, context, pool, leaked, onStarted, token) + .Finally(() => gate.Close(token)); + }); - private static IObservable RunCaseNow(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked, Action onStarted) + private static IObservable RunCaseNow(object? instance, Type cls, TestCase c, List output, TimeSpan deadline, MeshTestContext? context, IIoPool pool, List leaked, Action onStarted, object token) { if (c.Skip is not null) return Observable.Return(new CaseResult(cls.Name, c.Name, "⏭ skipped", c.Skip, TimeSpan.Zero)); @@ -162,6 +206,7 @@ private static IObservable RunCaseNow(object? instance, Type cls, Te { try { + CaseGate.Enter(token); if (context is not null) context.CancellationToken = ct; var result = c.Method.Invoke(instance, Arguments(c, ct)); diff --git a/test/Memex.Portal.Shared.Test/Memex.Portal.Shared.Test.csproj b/test/Memex.Portal.Shared.Test/Memex.Portal.Shared.Test.csproj index dba3c83c60..fb69b91381 100644 --- a/test/Memex.Portal.Shared.Test/Memex.Portal.Shared.Test.csproj +++ b/test/Memex.Portal.Shared.Test/Memex.Portal.Shared.Test.csproj @@ -50,5 +50,8 @@ + + diff --git a/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs b/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs new file mode 100644 index 0000000000..009d9d5951 --- /dev/null +++ b/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs @@ -0,0 +1,73 @@ +#pragma warning disable CS1591 + +using System; +using System.Linq; +using System.Reactive; +using System.Reactive.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using MeshWeaver.AI; // MeshOperations — its namespace is a frozen binary contract (#2370) +using MeshWeaver.Data; +using MeshWeaver.Fixture; +using MeshWeaver.Hosting.Monolith.TestBase; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using MeshWeaver.Testing.InMesh; +using Xunit; + +namespace Memex.Portal.Shared.Test; + +/// +/// RunTests (the run_tests tool, memex tests) keeps ONE subscription to a +/// node's streaming Tests area and turns its frames into an activity log: a line per case as +/// it runs and as it lands, the case's output, and a terminal status that is the verdict. +/// +/// Why it exists: rendering the area RUNS the suite, and a one-shot get …/area/Tests +/// opens a fresh subscription each time — polling it re-runs every live case (measured on +/// memex.meshweaver.cloud: each read filed the Maintenance suite's request nodes again). +/// +public class RunTestsStreamsIntoAnActivityTest(ITestOutputHelper output) : MonolithMeshTestBase(output) +{ + private const string Node = "RunTestsProbe"; + + protected override MeshBuilder ConfigureMesh(MeshBuilder builder) + => base.ConfigureMesh(builder) + .AddMeshNodes(TestUsers.SampleUsers()) + .AddMeshNodes(new MeshNode(Node) { Name = "Probe" }) + .ConfigureDefaultNodeHub(config => config.AddLayout(layout => layout + .WithView("Tests", (LayoutAreaHost host, RenderingContext _) => + MeshTestRunner.Area(host, "Probe", + [ + MeshTestCase.Live("slow", log => + { + log("contacted the service"); + return Observable.Timer(TimeSpan.FromMilliseconds(2500)).Select(_ => Unit.Default); + }), + MeshTestCase.Of("fails", () => throw new InvalidOperationException("the assertion message")), + ])))); + + [HubFact] + public async Task RunTests_LogsEveryCase_AndEndsWithTheVerdict() + { + var answer = await new MeshOperations(Mesh).RunTests("@" + Node, timeoutSeconds: 60) + .FirstAsync().Timeout(TestTimeouts.Convergence).Await(); + using var doc = JsonDocument.Parse(answer); + Assert.Equal("Dispatched", doc.RootElement.GetProperty("status").GetString()); + var activityPath = doc.RootElement.GetProperty("activityPath").GetString()!; + + var log = await Mesh.GetMeshNodeStream(activityPath) + .Select(node => node.ContentAs(Mesh.JsonSerializerOptions)) + .Where(l => l?.Status is ActivityStatus.Succeeded or ActivityStatus.Failed) + .FirstAsync() + .Timeout(TimeSpan.FromSeconds(60)).Await(); + + var lines = log!.Messages.Select(m => m.Message).ToList(); + Assert.Equal(ActivityStatus.Failed, log.Status); + Assert.Contains(lines, l => l.StartsWith("▶ slow", StringComparison.Ordinal) && l.Contains("contacted the service")); + Assert.Contains(lines, l => l.StartsWith("✅", StringComparison.Ordinal) && l.Contains("slow")); + Assert.Contains(lines, l => l.StartsWith("❌", StringComparison.Ordinal) && l.Contains("the assertion message")); + Assert.Contains(lines, l => l.Contains("1/2 passed", StringComparison.Ordinal)); + } +} diff --git a/test/MeshWeaver.Cli.Test/TestsCommandTest.cs b/test/MeshWeaver.Cli.Test/TestsCommandTest.cs index 5b67a59b6c..49879f41bb 100644 --- a/test/MeshWeaver.Cli.Test/TestsCommandTest.cs +++ b/test/MeshWeaver.Cli.Test/TestsCommandTest.cs @@ -4,63 +4,47 @@ namespace MeshWeaver.Cli.Test; /// -/// memex tests reads a node's Tests area frame by frame. Pinned over the PURE seam: which -/// frame is progress and which is the verdict, what a verdict says, and that a row is printed again -/// only when its state or output changed — never for a ticking clock. +/// memex tests prints a Tests run's ACTIVITY line by line (it never polls the Tests area, +/// which would re-run the suite on every read). Pinned over the pure seam: what an activity read +/// says, and which lines are new — including the lines that slid out of the window unseen. /// public class TestsCommandTest { - private const string Progress = """ - {"areas":{ - "\"$Menu:Node\"":{"items":[{"label":"Request approval","icon":"✅"}]}, - "\"Tests\"":{"$type":"StackControl","id":"tests-running","areas":[{"area":"Tests/Title"},{"area":"Tests/Cases"}]}, - "\"Tests/Title\"":{"$type":"HtmlControl","data":"

Store tests — 1 of 3 done, running for 4s

"}, - "\"Tests/Cases\"":{"$type":"DataGridControl","data":[ - {"class":"Store","case":"first","result":"✔","time":"0.1s","output":""}, - {"class":"Store","case":"slow","result":"▶","time":"3.9s","output":"contacted the service"}, - {"class":"Store","case":"last","result":"⏳","time":"","output":""}]}}} + private const string Running = """ + {"$type":"MeshNode","path":"Roland/_Activity/ab12cd34","content":{"$type":"ActivityLog", + "status":"Running","messageCount":3, + "messages":[{"message":"Run the Tests area of Store/Maintenance"},{"message":"▶ slow (1.0s) — contacted the service"},{"message":"✔ first (0.1s)"}]}} """; - private const string Verdict = """ - {"areas":{ - "\"Tests\"":{"$type":"StackControl","areas":[{"area":"Tests/Title"},{"area":"Tests/Cases"}]}, - "\"Tests/Title\"":{"$type":"HtmlControl","data":"

Store tests — 2/3 passed

"}, - "\"Tests/Cases\"":{"$type":"DataGridControl","data":[ - {"class":"Store","case":"first","result":"✅ pass","time":"0.1s","output":""}, - {"class":"Store","case":"slow","result":"❌ FAIL","time":"45.0s","output":"timed out: no verdict within 45s · contacted the service"}, - {"class":"Store","case":"last","result":"✅ pass","time":"0.0s","output":""}]}}} + private const string Failed = """ + {"$type":"MeshNode","path":"Roland/_Activity/ab12cd34","content":{"$type":"ActivityLog", + "status":"Failed","messageCount":6, + "messages":[{"message":"✔ first (0.1s)"},{"message":"❌ FAIL slow (45.0s) — timed out: no verdict within 45s"},{"message":"Store tests — 1/2 passed"},{"message":"Store tests: not every case passed."}]}} """; [Fact] - public void A_progress_frame_is_running_and_names_every_case() + public void An_activity_read_carries_its_status_count_and_window() { - var frame = TestsCommand.Parse(Progress); - Assert.True(frame.Running); - Assert.Equal("Store tests — 1 of 3 done, running for 4s", frame.Title); - Assert.Equal(["first", "slow", "last"], frame.Rows.Select(r => r.Case)); - Assert.Equal("contacted the service", frame.Rows[1].Output); + var read = TestsCommand.ParseActivity(Running); + Assert.Equal("Running", read.Status); + Assert.False(read.Terminal); + Assert.Equal(3, read.MessageCount); + Assert.Equal(3, read.Window.Length); } [Fact] - public void The_verdict_frame_fails_on_a_failed_case_and_the_chrome_never_counts() + public void Only_new_lines_print_and_archived_ones_are_announced() { - var frame = TestsCommand.Parse(Verdict); - Assert.False(frame.Running); - Assert.False(frame.Passed); + Assert.Equal(3, TestsCommand.NewLines(0, TestsCommand.ParseActivity(Running)).Length); + Assert.Empty(TestsCommand.NewLines(3, TestsCommand.ParseActivity(Running))); - var green = TestsCommand.Parse(Verdict.Replace("❌ FAIL", "✅ pass").Replace("2/3", "3/3")); - Assert.True(green.Passed); - } - - [Fact] - public void A_row_is_reprinted_only_when_its_state_or_output_changes() - { - var printed = new Dictionary(); - Assert.Equal(3, TestsCommand.Changes(printed, TestsCommand.Parse(Progress)).Count); - Assert.Empty(TestsCommand.Changes(printed, TestsCommand.Parse(Progress.Replace("3.9s", "4.9s")))); + var failed = TestsCommand.ParseActivity(Failed); + Assert.True(failed.Terminal); + Assert.Equal(["❌ FAIL slow (45.0s) — timed out: no verdict within 45s", "Store tests — 1/2 passed", "Store tests: not every case passed."], + TestsCommand.NewLines(3, failed)); - var verdictLines = TestsCommand.Changes(printed, TestsCommand.Parse(Verdict)); - Assert.Equal(3, verdictLines.Count); - Assert.Contains(verdictLines, l => l.Contains("timed out", StringComparison.Ordinal)); + var afterAGap = TestsCommand.NewLines(0, failed); + Assert.StartsWith("… 2 line(s) were archived", afterAGap[0]); + Assert.Equal(5, afterAGap.Length); } } diff --git a/test/MeshWeaver.Layout.Test/TestsAreaFrameTest.cs b/test/MeshWeaver.Layout.Test/TestsAreaFrameTest.cs new file mode 100644 index 0000000000..618f61ff2c --- /dev/null +++ b/test/MeshWeaver.Layout.Test/TestsAreaFrameTest.cs @@ -0,0 +1,55 @@ +using System.Collections.Immutable; +using System.Text.Json; +using MeshWeaver.Layout; +using Xunit; + +namespace MeshWeaver.Layout.Test; + +/// +/// reads a streamed Tests area off the wire for a consumer that is not +/// the page (MeshOperations.RunTests). Pinned: a progress frame is transient, chrome never +/// counts, two classes' same-named cases stay apart, and a ticking clock is not a change. +/// +public class TestsAreaFrameTest +{ + private static JsonElement Frame(string json) => JsonSerializer.Deserialize(json); + + private static readonly JsonElement Progress = Frame(""" + {"areas":{ + "\"$Menu:Node\"":{"items":[{"label":"Request approval","icon":"✅"}]}, + "\"Tests\"":{"$type":"StackControl","id":"tests-running","areas":[{"area":"Tests/Title"},{"area":"Tests/Cases"}]}, + "\"Tests/Title\"":{"$type":"HtmlControl","data":"

Store tests — 1 of 3 done, running for 4s

"}, + "\"Tests/Cases\"":{"$type":"DataGridControl","data":[ + {"class":"A","case":"same","result":"✔","time":"0.1s","output":""}, + {"class":"B","case":"same","result":"▶","time":"3.9s","output":"contacted the service"}, + {"class":"B","case":"last","result":"⏳","time":"","output":""}]}}} + """); + + [Fact] + public void A_progress_frame_is_transient_and_keeps_same_named_cases_apart() + { + var frame = TestsAreaFrame.Read(Progress); + Assert.True(frame.Materialized); + Assert.True(frame.Transient); + Assert.False(frame.Passed); + Assert.Equal("Store tests — 1 of 3 done, running for 4s", frame.Title); + Assert.Equal(3, frame.Rows.Select(r => r.Key).Distinct().Count()); + Assert.Empty(frame.Text); // the chrome's ✅ icon is not a verdict + } + + [Fact] + public void A_ticking_clock_is_not_a_change_and_a_verdict_is_read() + { + var (first, printed) = TestsAreaFrame.Changes(ImmutableDictionary.Empty, TestsAreaFrame.Read(Progress)); + Assert.Equal(3, first.Length); + var ticked = TestsAreaFrame.Read(Frame(Progress.GetRawText().Replace("3.9s", "4.9s"))); + Assert.Empty(TestsAreaFrame.Changes(printed, ticked).Changed); + + var verdict = TestsAreaFrame.Read(Frame(Progress.GetRawText() + .Replace("\"id\":\"tests-running\",", "") + .Replace("1 of 3 done, running for 4s", "3/3 passed") + .Replace("✔", "✅ pass").Replace("▶", "✅ pass").Replace("⏳", "✅ pass"))); + Assert.False(verdict.Transient); + Assert.True(verdict.Passed); + } +} From a2765ee3a76b3da2782bcc869458e4fafd7541f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:25:34 +0200 Subject: [PATCH 3/5] =?UTF-8?q?feat(api):=20POST=20api/mesh/run-tests=20?= =?UTF-8?q?=E2=80=94=20the=20route=20memex=20tests=20calls=20(Bearer-only:?= =?UTF-8?q?=20it=20executes)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) --- memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs | 6 ++++++ test/Memex.Portal.Shared.Test/MeshApiCookieAuthTest.cs | 1 + 2 files changed, 7 insertions(+) diff --git a/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs b/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs index 893faca0a9..4784d10857 100644 --- a/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs +++ b/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs @@ -146,6 +146,12 @@ public static IEndpointRouteBuilder MapMeshApi(this IEndpointRouteBuilder endpoi group.MapPost("/execute-script", (HttpContext http, IMessageHub rootHub, ExecuteScriptBody body, CancellationToken ct) => RunString(http, rootHub, ct, ops => ops.ExecuteScript(body.Path, body.TimeoutSeconds ?? 120))); + // Runs a node's Tests area as an ACTIVITY (one area subscription for the whole run) and + // answers {status, activityPath} at once — `memex tests` polls the activity, never the area, + // because every render of a Tests area runs the suite again. + group.MapPost("/run-tests", (HttpContext http, IMessageHub rootHub, ExecuteScriptBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.RunTests(body.Path, body.TimeoutSeconds ?? MeshOperations.DefaultRunTestsSeconds))); + // First-full-frame render of a layout area — the SSR seeding verb (portal-next): // returns {areas, data} EXACTLY as the sync-stream wire delivers it. Read-only, and // on the cookie-or-Bearer policy: this is the verb an SSR page render is FOR. diff --git a/test/Memex.Portal.Shared.Test/MeshApiCookieAuthTest.cs b/test/Memex.Portal.Shared.Test/MeshApiCookieAuthTest.cs index 3168ba6c26..54621d6266 100644 --- a/test/Memex.Portal.Shared.Test/MeshApiCookieAuthTest.cs +++ b/test/Memex.Portal.Shared.Test/MeshApiCookieAuthTest.cs @@ -74,6 +74,7 @@ public class MeshApiCookieAuthTest "/api/mesh/recycle", "/api/mesh/compile", "/api/mesh/execute-script", + "/api/mesh/run-tests", "/api/mesh/mirror", "/api/mesh/upload", ]; From a28dbad3ef833e7b067393ae110ecc1b3ee6eb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:25:49 +0200 Subject: [PATCH 4/5] docs(testing): name where the run-tests route and the run_tests tool live Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Data/Architecture/WritingTests.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md b/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md index 2dc6cd6674..31ec3cbd22 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md @@ -184,7 +184,9 @@ case from a stuck one and nothing naming the case. It now streams: with `{status: "Dispatched", activityPath}`; poll `get @{activityPath}`, which starts nothing. `memex tests @` is that loop in a terminal (`POST api/mesh/run-tests`, then the activity every two seconds), exiting `0` (passed), `1` (failed) or `4` (no terminal status within `--timeout`). The - portal route and the MCP `run_tests` tool live with the portal in MeshWeaver.Plugins. + route is `Memex.Portal.Shared`'s `MeshApiEndpoints` (Bearer-only, it executes); agents reach the + same operation through the MCP `run_tests` tool, which lives with the MCP surface in + MeshWeaver.Plugins. ## The Canonical Test Base From 1acc7c17734032a642f74e81e63a24ac03cf9e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:49:17 +0200 Subject: [PATCH 5/5] fix(testing): RunTests activity lines are keyed (activity.tests.case/caseOutput/summary) The unkeyed-LogMessage ratchet refused the three verbatim lines. Case lines and the verdict summary are now catalog-keyed (en/de); a failing verdict's own words ride the failure line. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/MeshWeaver.Layout/TestsAreaFrame.cs | 14 +++++++ .../MeshOperations.RunTests.cs | 37 ++++++++++++------- .../Localization/strings.de.json | 3 ++ .../Localization/strings.en.json | 3 ++ .../RunTestsStreamsIntoAnActivityTest.cs | 2 +- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/MeshWeaver.Layout/TestsAreaFrame.cs b/src/MeshWeaver.Layout/TestsAreaFrame.cs index 731ef9acd9..5dfe4d98b6 100644 --- a/src/MeshWeaver.Layout/TestsAreaFrame.cs +++ b/src/MeshWeaver.Layout/TestsAreaFrame.cs @@ -60,6 +60,20 @@ public bool Passed } } + /// + /// How many counted cases passed, out of how many: the verdict's own N/M passed when it + /// carries one, otherwise the rows (✅ over every row that is not ⏭ skipped). + /// + public (int Passed, int Total) Counts() + { + var summary = Text.Prepend(Title ?? "").Select(s => PassSummary.Match(s)).FirstOrDefault(m => m.Success); + if (summary is not null) + return (int.Parse(summary.Groups[1].Value, System.Globalization.CultureInfo.InvariantCulture), + int.Parse(summary.Groups[2].Value, System.Globalization.CultureInfo.InvariantCulture)); + var counted = Rows.Where(r => !r.Result.StartsWith('⏭')).ToImmutableArray(); + return (counted.Count(r => r.Result.StartsWith('✅')), counted.Length); + } + /// Reads one serialized area store for the area . /// The frame as the sync stream carries it. /// The area name (normally Tests). diff --git a/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs b/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs index b8bc312bde..819905b005 100644 --- a/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs +++ b/src/MeshWeaver.Mesh.Operations/MeshOperations.RunTests.cs @@ -110,11 +110,16 @@ private IObservable WatchTestsArea( { if (frame.NotFound) throw new InvalidOperationException($"{nodePath} has no Tests area."); - foreach (var line in frame.Text) - ctx.Log(new LogMessage(line, LogLevel.Information)); + var (passed, total) = frame.Counts(); + ctx.Log(new LogMessage($"{passed} of {total} cases passed", + passed == total ? LogLevel.Information : LogLevel.Warning) + .WithKey("activity.tests.summary", ("passed", (object?)passed), ("total", (object?)total))); if (!frame.Passed) - throw new InvalidOperationException( - $"{frame.Title ?? nodePath + " tests"}: not every case passed."); + // The area's own words (its title and, for a suite rendered as markdown, its + // failing rows) — upstream text, carried verbatim by the activity's failure line. + throw new InvalidOperationException(string.Join(" · ", + new[] { frame.Title ?? nodePath + " tests" } + .Concat(frame.Rows.Length == 0 ? frame.Text.Where(t => t.Contains('❌')) : []))); return Unit.Default; }); }); @@ -142,15 +147,21 @@ private IObservable WatchTestsFrames( (state, frame) => { var (changed, printed) = TestsAreaFrame.Changes(state.Printed, frame); - foreach (var row in changed) - ctx.Log(new LogMessage(TestsAreaFrame.Line(row), - row.Result.StartsWith('❌') || row.Result.StartsWith('✖') ? LogLevel.Warning : LogLevel.Information)); + foreach (var row in changed.Where(r => r.Result != "⏳")) + ctx.Log(CaseLine(row)); return (frame, printed); }) - .Select(state => state.Frame!) - .Do(frame => - { - if (!frame.Transient && frame.Title is { } title) - ctx.Log(new LogMessage(title, LogLevel.Information)); - }); + .Select(state => state.Frame!); + + // One case as a keyed activity line: the viewer's language renders the frame, the case's own + // name and output stay as written. + private static LogMessage CaseLine(TestsAreaFrame.Row row) + { + var level = row.Result.StartsWith('❌') || row.Result.StartsWith('✖') ? LogLevel.Warning : LogLevel.Information; + if (row.Output.Length == 0) + return new LogMessage(TestsAreaFrame.Line(row), level) + .WithKey("activity.tests.case", ("result", (object?)row.Result), ("case", (object?)row.Case), ("time", (object?)row.Time)); + return new LogMessage(TestsAreaFrame.Line(row), level) + .WithKey("activity.tests.caseOutput", ("result", (object?)row.Result), ("case", (object?)row.Case), ("time", (object?)row.Time), ("output", (object?)row.Output)); + } } diff --git a/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json b/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json index 8825efb778..40edf9a0d7 100644 --- a/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json +++ b/src/MeshWeaver.Messaging.Hub/Localization/strings.de.json @@ -1718,6 +1718,9 @@ "ui.threadQueueStateActionFailed": "Nicht ausführbar", "ui.threadQueueStateFailed": "Fehlgeschlagen", "activity.tests.title": "Tests von {path} ausführen", + "activity.tests.case": "{result} {case} ({time})", + "activity.tests.caseOutput": "{result} {case} ({time}) — {output}", + "activity.tests.summary": "{passed} von {total} Fällen bestanden", "tests.progress":"{0}-Tests — {1} von {2} erledigt, seit {3}s in Arbeit", "tests.column.class": "Klasse", "tests.column.case": "Fall", diff --git a/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json b/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json index 4c59338ebe..48918d1446 100644 --- a/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json +++ b/src/MeshWeaver.Messaging.Hub/Localization/strings.en.json @@ -1718,6 +1718,9 @@ "ui.threadQueueStateActionFailed": "Could not act", "ui.threadQueueStateFailed": "Failed", "activity.tests.title": "Run the Tests area of {path}", + "activity.tests.case": "{result} {case} ({time})", + "activity.tests.caseOutput": "{result} {case} ({time}) — {output}", + "activity.tests.summary": "{passed} of {total} cases passed", "tests.progress":"{0} tests — {1} of {2} done, running for {3}s", "tests.column.class": "Class", "tests.column.case": "Case", diff --git a/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs b/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs index 009d9d5951..bdb0b6a830 100644 --- a/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs +++ b/test/Memex.Portal.Shared.Test/RunTestsStreamsIntoAnActivityTest.cs @@ -68,6 +68,6 @@ public async Task RunTests_LogsEveryCase_AndEndsWithTheVerdict() Assert.Contains(lines, l => l.StartsWith("▶ slow", StringComparison.Ordinal) && l.Contains("contacted the service")); Assert.Contains(lines, l => l.StartsWith("✅", StringComparison.Ordinal) && l.Contains("slow")); Assert.Contains(lines, l => l.StartsWith("❌", StringComparison.Ordinal) && l.Contains("the assertion message")); - Assert.Contains(lines, l => l.Contains("1/2 passed", StringComparison.Ordinal)); + Assert.Contains(lines, l => l.Contains("1 of 2 cases passed", StringComparison.Ordinal)); } }