From 5ab170c0648e33a60c24162f06aa1c6bdf9e0f0b Mon Sep 17 00:00:00 2001 From: ejc3 Date: Mon, 26 Oct 2026 17:15:02 +0000 Subject: [PATCH] fleet/web mock: skip whole frames when behind, the same ones for every stream main's Fleet Web job failed "switching streams" on 47ec5fcf: after the switch back to direct, ui.shownMode stayed "aggregated" for 20 s; the same commit's previous run, and #1332's runs, show the same intermittent failure. Measured on the CI runner with the switch logged every 0.5 s (run 40365237969): during the switch the mock serves the 3 aggregates and the 15 direct tiles, 18 streams and about 1.75 times the pixels of direct alone. It fell behind (about 60 renders queued) and every stream got about 6 of its 10 frames a second, but each stream a different 6: each lane skipped a frame when its own previous draw was still in flight. The switch needs one t that all 15 direct tiles hold, and with each missing 4 frames in 10 independently, no such t came in 22 s. The two-viewers timeout fixed by #1339 (11 of 15 tiles synced) is the same mechanism: a tile without the common t is drawn behind, not synced. The renderer now skips a frame for every stream at once when the last frame's draws have not all started (the pool's backlog). Behind, it serves fewer frames, and all streams hold the same t. With this change the same runner switched back to direct within 0.6 s in each of 13 switches (run 40366842617), passed all 49 Chromium tests, and the mock used 1.1 to 2.1 cores of its 3. - mock/test/mock.test.ts: one render thread asked for eight 360-high tiles at 30 fps; every frame number it sends reaches all eight streams. On the previous renderer: 0 of 71 frame numbers reached all eight. - ci.yml: the Fleet Web job's comment says what a switch costs and how the mock behaves when behind. --- .github/workflows/ci.yml | 7 ++++-- fleet/web/mock/renderer.ts | 8 +++++++ fleet/web/mock/test/mock.test.ts | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b28632e87..d0e45d325 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,8 +150,11 @@ jobs: # mock used about 2.2 cores of its 3 at 10 frames per second through the suite. # Before the render loop lost its per-pixel allocations it needed more than 3, # dropped frames from the first test, and later tests timed out waiting for - # every tile at one common t. Smaller screens leave tiles too few pixels for the - # seam checks' colour fit. + # every tile at one common t. A stream switch serves the aggregates and the + # direct tiles at once (18 streams, about 1.75 times the pixels) for a second or + # so; the mock then skips whole frames, the same ones for every stream, so the + # page still finds a t every tile holds. Smaller screens leave tiles too few + # pixels for the seam checks' colour fit. FLEET_E2E_VIEW: "960x600" FLEET_MOCK_FPS: "10" FLEET_MOCK_THREADS: "3" diff --git a/fleet/web/mock/renderer.ts b/fleet/web/mock/renderer.ts index 2731f492d..463e272f5 100644 --- a/fleet/web/mock/renderer.ts +++ b/fleet/web/mock/renderer.ts @@ -348,6 +348,14 @@ export class MockRenderers { for (const s of this.subs) if (s.tile === id) this.resize(s, s.reqH); } this.stats.renderBacklog = this.pool.backlog; + // Behind its clock (the last frame's draws have not all started), the renderer skips + // this frame for every stream at once. Skipping per stream would leave each tile a + // different subset of t, and a page draws all of them at one t: with 15 tiles each + // missing 4 frames in 10, no t is held by all 15, and the page never shows them in sync. + if (this.pool.backlog > 0) { + for (const s of this.subs) if (s.encoder) this.stats.framesSkipped++; + return; + } const groups = new Map(); for (const s of this.subs) { if (!s.encoder) continue; diff --git a/fleet/web/mock/test/mock.test.ts b/fleet/web/mock/test/mock.test.ts index 67c2f11db..b758e9051 100644 --- a/fleet/web/mock/test/mock.test.ts +++ b/fleet/web/mock/test/mock.test.ts @@ -744,3 +744,41 @@ describe("the mock speaks the coordinator's snapshot shapes", () => { expect([...mock].filter((p) => p !== "" && !allowed(p)).sort()).toEqual([]); }); }); + +describe("mock renderer, more streams than it can draw", () => { + it("drops whole frames: every stream gets the same frame numbers, so the page can draw them at one t", async () => { + // One render thread asked for eight 360-high tiles at 30 fps: it draws a few frames a + // second. A renderer behind on its clock skips frames for every tile at once; skipping + // per stream gives each tile a different subset of t, and the page, which draws every + // tile at one common t, then has no t that all of them hold. + const scene = coord.patternScene(); + const r = new MockRenderers({ port: 0, publicHost: "localhost", fps: 30, threads: 1, maxHeight: 360, epochUnixMs: scene.epoch_unix_ms, scene: () => scene }); + await r.listen(); + const ids = Array.from({ length: 8 }, (_, i) => `t${i}`); + ids.forEach((id, i) => r.setTile(id, { x: (i % 4) / 4, y: Math.floor(i / 4) / 2, w: 0.25, h: 0.5 })); + const got = new Map(ids.map((id) => [id, []])); + const sockets = ids.map((id) => { + const ws = new WebSocket(`${r.frameUrl(id)}?h=360`); + ws.on("message", (d, bin) => { + if (!bin) return; + const p = parseFrame(new Uint8Array(d as Buffer)); + if (p.ok) got.get(id)!.push(p.frame.frame); + }); + return ws; + }); + await new Promise((res) => setTimeout(res, 4000)); + for (const ws of sockets) ws.close(); + await r.close(); + // Frames after the first second, once every encoder has started. + const first = Math.max(...ids.map((id) => got.get(id)![0] ?? Infinity)); + const sets = ids.map((id) => new Set(got.get(id)!.filter((n) => n >= first + 30))); + const all = new Set(); + for (const s of sets) for (const n of s) all.add(n); + const everywhere = [...all].filter((n) => sets.every((s) => s.has(n))); + // It is behind (far fewer than 30 frames a second) ... + expect(all.size).toBeGreaterThan(0); + expect(all.size).toBeLessThan(2.5 * 30); + // ... and still every frame number it sent, it sent on every stream (the last one may be in flight). + expect(everywhere.length, `frames on all 8 streams: ${everywhere.length} of ${all.size}`).toBeGreaterThanOrEqual(all.size - 1); + }, 20_000); +});