feat(testing): Tests areas stream per-case progress, bound every case, and get a console (memex tests)

memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs

@@ -146,6 +146,12 @@ public static IEndpointRouteBuilder MapMeshApi(this IEndpointRouteBuilder endpoi
         group.MapPost("/execute-script", (HttpContext http, IMessageHub rootHub, ExecuteScriptBody body, CancellationToken ct) =>
             RunString(http, rootHub, ct, ops => ops.ExecuteScript(body.Path, body.TimeoutSeconds ?? 120)));
 
+        // Runs a node's Tests area as an ACTIVITY (one area subscription for the whole run) and
+        // answers {status, activityPath} at once — `memex tests` polls the activity, never the area,
+        // because every render of a Tests area runs the suite again.
+        group.MapPost("/run-tests", (HttpContext http, IMessageHub rootHub, ExecuteScriptBody body, CancellationToken ct) =>
+            RunString(http, rootHub, ct, ops => ops.RunTests(body.Path, body.TimeoutSeconds ?? MeshOperations.DefaultRunTestsSeconds)));
+
         // First-full-frame render of a layout area — the SSR seeding verb (portal-next):
         // returns {areas, data} EXACTLY as the sync-stream wire delivers it. Read-only, and
         // on the cookie-or-Bearer policy: this is the verb an SSR page render is FOR.

src/MeshWeaver.Cli/MemexClient.cs

@@ -53,6 +53,8 @@ public Task<string> Recycle(string path, string? reason, CancellationToken ct) =
     public Task<string> Compile(string path, CancellationToken ct) => Post("api/mesh/compile", new { path }, ct);
     public Task<string> Diagnostics(string path, CancellationToken ct) => Post("api/mesh/diagnostics", new { path }, ct);
     public Task<string> ExecuteScript(string path, int timeoutSeconds, CancellationToken ct) => Post("api/mesh/execute-script", new { path, timeoutSeconds }, ct);
+    /// <summary>Runs a node's Tests area as an activity (<c>MeshOperations.RunTests</c>); answers <c>{status, activityPath}</c>.</summary>
+    public Task<string> RunTests(string path, int timeoutSeconds, CancellationToken ct) => Post("api/mesh/run-tests", new { path, timeoutSeconds }, ct);
     public Task<string> NavigateTo(string path, CancellationToken ct) => Post("api/mesh/navigate-to", new { path }, ct);
     public Task<string> BaseUrl(CancellationToken ct) => Post("api/mesh/base-url", new { }, ct);
 

src/MeshWeaver.Cli/Program.cs

@@ -165,6 +165,32 @@ async Task<int> Run(
     root.Subcommands.Add(cmd);
 }
 
+// --- tests -----------------------------------------------------------------
+{
+    var pathArg = new Argument<string>("path") { Description = "Node whose Tests area to run (e.g. @Admin/Maintenance/x)." };
+    var timeoutOpt = new Option<int>("--timeout") { Description = "Seconds to wait for the verdict.", DefaultValueFactory = _ => 600 };
+    var intervalOpt = new Option<int>("--interval") { Description = "Seconds between reads of the area.", DefaultValueFactory = _ => 2 };
+    var cmd = new Command("tests", "Run a node's Tests area and stream its progress until the verdict (exit 0 = all passed, 1 = a failure, 4 = no verdict in time).")
+        { pathArg, timeoutOpt, intervalOpt };
+    cmd.SetAction(async (result, ct) =>
+    {
+        try
+        {
+            var cfg = MemexConfig.Resolve(result.GetValue(baseUrlOpt), result.GetValue(tokenOpt));
+            using var client = new MemexClient(cfg);
+            return await TestsCommand.Run(client, result.GetValue(pathArg)!,
+                TimeSpan.FromSeconds(result.GetValue(timeoutOpt)), TimeSpan.FromSeconds(Math.Max(1, result.GetValue(intervalOpt))),
+                Console.Out, ct);
+        }
+        catch (MemexCliException ex)
+        {
+            await Console.Error.WriteLineAsync(ex.Message);
+            return 2;
+        }
+    });
+    root.Subcommands.Add(cmd);
+}
+
 // --- upload ----------------------------------------------------------------
 {
     var pathArg = new Argument<string>("path") { Description = "Target mesh path {nodePath}/{collection}/{filePath}." };

src/MeshWeaver.Cli/TestsCommand.cs

@@ -0,0 +1,113 @@
+using System.Collections.Immutable;
+using System.Text.Json;
+
+namespace MeshWeaver.Cli;
+
+/// <summary>
+/// <c>memex tests &lt;path&gt;</c> — run one node's <c>Tests</c> area and print its progress as a
+/// console until the verdict.
+///
+/// <para>🚨 It does NOT poll <c>get @path/area/Tests</c>. Rendering the area RUNS the suite, and every
+/// one-shot read opens a fresh subscription — measured on memex.meshweaver.cloud, each read filed the
+/// Maintenance suite's live request nodes again, and answered with whatever verdict an earlier
+/// subscription had cached. Instead the portal runs the area ONCE, holding one subscription, as an
+/// activity (<c>POST api/mesh/run-tests</c> → <c>MeshOperations.RunTests</c>) that logs a line per
+/// case as it starts, writes output and lands; this command polls that ACTIVITY node, which starts
+/// nothing, and prints each new line.</para>
+///
+/// <para>Exit codes: <c>0</c> the activity Succeeded (every counted case passed), <c>1</c> it Failed
+/// (a case failed, the area has none, or the portal's own bound elapsed — its last line names the
+/// case still running), <c>4</c> no terminal status within <c>--timeout</c>.</para>
+/// </summary>
+public static class TestsCommand
+{
+    /// <summary>One read of the activity node.</summary>
+    /// <param name="Status">The activity status (<c>Running</c>, <c>Succeeded</c>, <c>Failed</c>, …).</param>
+    /// <param name="MessageCount">The true number of lines ever written.</param>
+    /// <param name="Window">The most recent lines (older ones are archived under <c>{activity}/_Log</c>).</param>
+    public sealed record ActivityRead(string Status, int MessageCount, ImmutableArray<string> Window)
+    {
+        /// <summary>True once the run has a terminal status.</summary>
+        public bool Terminal => Status is not ("Running" or "Pending" or "");
+    }
+
+    /// <summary>Reads a <c>get @activity</c> answer. Pure over the JSON.</summary>
+    /// <param name="json">The activity node.</param>
+    public static ActivityRead ParseActivity(string json)
+    {
+        using var doc = JsonDocument.Parse(json);
+        var content = doc.RootElement.TryGetProperty("content", out var c) ? c : doc.RootElement;
+        var status = content.TryGetProperty("status", out var s) && s.ValueKind == JsonValueKind.String ? s.GetString() ?? "" : "";
+        var messages = content.TryGetProperty("messages", out var m) && m.ValueKind == JsonValueKind.Array
+            ? m.EnumerateArray()
+                .Select(e => e.TryGetProperty("message", out var text) ? text.GetString() ?? "" : "")
+                .ToImmutableArray()
+            : [];
+        var count = content.TryGetProperty("messageCount", out var n) && n.TryGetInt32(out var value) ? value : messages.Length;
+        return new ActivityRead(status, count, messages);
+    }
+
+    /// <summary>
+    /// The lines of <paramref name="read"/> not yet printed, given that <paramref name="printed"/>
+    /// lines were printed before — plus a note when some slid out of the window unseen.
+    /// </summary>
+    /// <param name="printed">How many lines have been printed so far.</param>
+    /// <param name="read">The activity just read.</param>
+    public static ImmutableArray<string> NewLines(int printed, ActivityRead read)
+    {
+        var fresh = read.MessageCount - printed;
+        if (fresh <= 0)
+            return [];
+        var shown = Math.Min(fresh, read.Window.Length);
+        var lines = read.Window.Skip(read.Window.Length - shown);
+        return fresh > shown
+            ? [$"… {fresh - shown} line(s) were archived before they could be read (see the activity's _Log)", .. lines]
+            : [.. lines];
+    }
+
+    /// <summary>Starts the run, then prints the activity's lines until it is terminal; returns the exit code.</summary>
+    /// <param name="client">The portal client.</param>
+    /// <param name="path">The node whose Tests area to run.</param>
+    /// <param name="timeout">How long to wait for a terminal status.</param>
+    /// <param name="interval">How often to read the activity.</param>
+    /// <param name="output">Where the console goes.</param>
+    /// <param name="ct">Cancels the wait.</param>
+    public static async Task<int> Run(MemexClient client, string path, TimeSpan timeout, TimeSpan interval, TextWriter output, CancellationToken ct)
+    {
+        // The command's --timeout bounds EVERY request too, so a hung read cannot outlive it.
+        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        deadline.CancelAfter(timeout);
+        try
+        {
+            var started = await client.RunTests(path, (int)Math.Ceiling(timeout.TotalSeconds), deadline.Token);
+            using var dispatched = JsonDocument.Parse(started);
+            if (dispatched.RootElement.TryGetProperty("status", out var s) && s.GetString() != "Dispatched"
+                || !dispatched.RootElement.TryGetProperty("activityPath", out var activity))
+            {
+                await output.WriteLineAsync(started);
+                return 1;
+            }
+            var activityPath = activity.GetString()!;
+            await output.WriteLineAsync($"== {activityPath}");
+            var printed = 0;
+            while (true)
+            {
+                var read = ParseActivity(await client.Get("@" + activityPath, deadline.Token));
+                foreach (var line in NewLines(printed, read))
+                    await output.WriteLineAsync(line);
+                printed = Math.Max(printed, read.MessageCount);
+                if (read.Terminal)
+                {
+                    await output.WriteLineAsync($"== {read.Status}");
+                    return read.Status == "Succeeded" ? 0 : 1;
+                }
+                await Task.Delay(interval, deadline.Token);
+            }
+        }
+        catch (OperationCanceledException) when (deadline.IsCancellationRequested && !ct.IsCancellationRequested)
+        {
+            await output.WriteLineAsync($"no terminal status within {timeout.TotalSeconds:F0}s");
+            return 4;
+        }
+    }
+}

src/MeshWeaver.Data.Contract/ActivityCategory.cs

@@ -27,6 +27,12 @@ public static class ActivityCategory
     /// </summary>
     public const string WriteConflict = nameof(WriteConflict);
 
+    /// <summary>
+    /// Activity carrying a run of a node's <c>Tests</c> area: one line per case as it starts,
+    /// writes output and reaches its verdict (<c>run_tests</c> / <c>memex tests</c>).
+    /// </summary>
+    public const string TestRun = nameof(TestRun);
+
     /// <summary>
     /// Activity whose category is unknown or unclassified.
     /// </summary>

src/MeshWeaver.Documentation/Data/Architecture/DecentralisedTests.md

@@ -336,7 +336,8 @@ differently. Conflating them is the main way this design goes wrong.
 
 `PluginGateRunner` + `AreaProbe`: stands up a mesh, imports content, renders each type's `Tests`
 layout area over the ordinary client sync stream, and classifies the frames
-(`AreaProbe.ClassifyTestsFrame`): compile-progress and *"Area not found"* are transient; `❌` is red;
+(`AreaProbe.ClassifyTestsFrame`): compile-progress, a streamed Tests progress frame
+(`tests-running`, see [Writing Tests](../WritingTests)) and *"Area not found"* are transient; `❌` is red;
 `N/M passed` is green iff `N == M`; **no verdict inside the timeout is RED** — *"a Tests area that
 reports nothing is a broken gate, never a silent pass."*
 

src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md

@@ -144,6 +144,50 @@ are counted as `needs-mesh` and run by the gate, seeded from the build's output.
 `MeshWeaver.Fixture` and the two TestBase assemblies are this repo's OWN test support: they live
 under `test/` and are never packed or published.
 
+### A `Tests` area streams its progress — and `memex tests` is its console
+
+Rendering a node's `Tests` area RUNS its cases (`MeshWeaver.Testing.InMesh.MeshTestRunner`). The
+area used to emit **one frame, at the end**, so a case that called an external service live — or a
+case that hung — left the page on *"Rendering …"* for the whole suite, with nothing to tell a slow
+case from a stuck one and nothing naming the case. It now streams:
+
+| Frame | Shows | Carries |
+|---|---|---|
+| first, at once | every case ⏳ pending | `Id = tests-running` |
+| once a second while a case runs | the running case ▶ with its ticking elapsed time and every output line it has written so far; finished cases ✔ / ✖ | `Id = tests-running` |
+| last | the verdict: title `<suite> tests — N/M passed`, rows ✅ / ❌ / ⏭ with time and output | no id |
+
+- **Every case is bounded.** A case past its bound fails as `timed out: no verdict within Ns` and the
+  run moves on; a synchronous case runs as one leaf on the mesh's `Tests` I/O pool, so a body that
+  never returns costs its own bound, never the render thread.
+- **The verdict glyphs are reserved for the verdict frame.** A progress frame shows ✔ / ✖ and counts
+  cases *done*, never ✅ / ❌ or *"N/M passed"* — and it carries
+  `AreaFrameClassifier.TestsRunningId`, which the plugin gate (`AreaProbe`) and
+  `AreaFrameClassifier.IsTransientFrame` treat as *keep waiting*. A gate that latched a progress frame
+  would green a suite whose later cases had not run yet; a run cut off by the gate's timeout reports
+  the last progress it saw.
+- **Two ways to list cases.** `[MeshFact]` classes: `MeshTestRunner.Area(host, suite, assembly)`.
+  Static methods: `MeshTestRunner.Area(host, suite, cases)` over `MeshTestCase.Of(name, method)`
+  (synchronous) and `MeshTestCase.Live(name, () => observable, timeout)` (hosted, passes on first
+  emission); either overload hands the body a line writer whose output streams into the row.
+- **From a terminal or an agent — run it as an ACTIVITY, never by polling the area.** 🚨 Rendering
+  the area RUNS the suite, and every one-shot `get @<node>/area/Tests` opens a fresh subscription.
+  Measured on memex.meshweaver.cloud (`Admin/Maintenance/refresh-app-tiles-20260828-mainnode-3`): two
+  `get` reads 34 s apart each filed a new pair of the Maintenance suite's live request nodes
+  (`Admin/Maintenance/mnt-…`, 10:51:00Z and 10:51:34Z), and the second read answered in 0.9 s with the
+  verdict an EARLIER subscription had cached while its own run had only just begun. So a poll of the
+  area re-runs every live case and still does not show the run it started.
+  `MeshOperations.RunTests(path)` holds ONE subscription for the whole run and writes it to an
+  activity in the caller's own partition (category `TestRun`): a line per case as it runs and as it
+  lands, with its output; `Succeeded` when every counted case passed, `Failed` otherwise, or when no
+  verdict arrived within the bound (the last line names the case still running). It answers at once
+  with `{status: "Dispatched", activityPath}`; poll `get @{activityPath}`, which starts nothing.
+  `memex tests @<node>` is that loop in a terminal (`POST api/mesh/run-tests`, then the activity every
+  two seconds), exiting `0` (passed), `1` (failed) or `4` (no terminal status within `--timeout`). The
+  route is `Memex.Portal.Shared`'s `MeshApiEndpoints` (Bearer-only, it executes); agents reach the
+  same operation through the MCP `run_tests` tool, which lives with the MCP surface in
+  MeshWeaver.Plugins.
+
 ## The Canonical Test Base
 
 Every monolith test inherits `MonolithMeshTestBase`. The shape is always the same:

src/MeshWeaver.Layout/AreaFrameClassifier.cs

@@ -88,6 +88,22 @@ public static class AreaFrameClassifier
     /// </summary>
     public const string StorageUnavailableId = "storage-unavailable";
 
+    /// <summary>
+    /// <see cref="UiControl.Id"/> of every PROGRESS frame a <c>Tests</c> area serves while its
+    /// cases are still running (<c>MeshTestRunner.Area</c>): the table shows each case pending,
+    /// running or finished, with its elapsed time and its output so far. TRANSIENT.
+    ///
+    /// <para>The SIXTH state. A Tests area used to emit exactly ONE frame, at the end, so a case
+    /// that called a slow external service kept the page on "Rendering …" for its whole budget,
+    /// indistinguishable from a hung one. Streaming the progress fixes the page, and this id is
+    /// what keeps the stream safe for a consumer that reads a VERDICT off the first frame it can
+    /// classify (the plugin gate's <c>AreaProbe</c>): a progress frame is a promise that the
+    /// verdict frame follows, never a verdict of its own — its rows already carry ✔ for a case
+    /// that passed so far, and read as a verdict that would green a suite whose later cases have
+    /// not run yet.</para>
+    /// </summary>
+    public const string TestsRunningId = "tests-running";
+
     // The pre-id signal, kept as a fallback so a frame that lost its id on the way here (an
     // older peer, a control rebuilt from partial JSON) is still recognised. Never localize:
     // BuildNotFoundControl is deliberately English — it is a framework diagnostic, not UI copy.
@@ -141,16 +157,26 @@ public static bool IsHubRecycling(UiControl? control)
     public static bool IsStorageUnavailable(UiControl? control)
         => HasFrameId(control, StorageUnavailableId);
 
+    /// <summary>
+    /// True for a PROGRESS frame of a <c>Tests</c> area whose cases are still running — the
+    /// verdict frame replaces it once the last case finished or timed out.
+    /// </summary>
+    /// <param name="control">The rendered control, or <c>null</c>.</param>
+    public static bool IsTestsRunning(UiControl? control)
+        => HasFrameId(control, TestsRunningId);
+
     /// <summary>
     /// True for a frame that is not the area's content and will be REPLACED without anyone
-    /// acting: the compile-progress page, and the <see cref="RedirectControl"/> it emits once
-    /// the build settles. The single predicate a waiter needs — "keep waiting, this is not the
-    /// answer". A genuinely missing area (<see cref="IsAreaNotFound"/>) is deliberately NOT
-    /// transient: nothing is going to replace it.
+    /// acting: the compile-progress page, the <see cref="RedirectControl"/> it emits once
+    /// the build settles, and a Tests area's progress frame, which its verdict frame replaces.
+    /// The single predicate a waiter needs — "keep waiting, this is not the answer". A genuinely
+    /// missing area (<see cref="IsAreaNotFound"/>) is deliberately NOT transient: nothing is
+    /// going to replace it.
     /// </summary>
     /// <param name="control">The rendered control, or <c>null</c>.</param>
     public static bool IsTransientFrame(UiControl? control)
-        => IsCompileProgress(control) || IsHubRecycling(control) || control is RedirectControl;
+        => IsCompileProgress(control) || IsHubRecycling(control) || IsTestsRunning(control)
+           || control is RedirectControl;
 
     // UiControl.Id is `object?`, so a frame that came back over the sync stream carries it as
     // whatever the deserializer produced for a JSON string (a JsonElement, not a string). Compare