From fc864653024ed078ef84d15c76f106ce56117d31 Mon Sep 17 00:00:00 2001 From: Jonas Templestein <242550+jonastemplestein@users.noreply.github.com> Date: Sun, 22 Nov 2026 11:33:41 +0000 Subject: [PATCH] The bridges connect again when their connection to the mind ends A session's sign-in lasts 4 hours. The live bridge and the device-events bridge connected once, so after 4 hours the Mac heard no watcher and its presence events were lost. Both now hold their connection through `bridge/mind-connection.mjs`: it notices a closed or failed connection, connects again with the stored sign-in after 1 s, doubling to 60 s, and runs each bridge's setup again (the live bridge subscribes again and asks for the ICE servers again; it checks once a minute that its subscription still hears). Events wait during a gap and go up in order; after a reconnect the app publishes the present presence again. `live.state().signalling` says whether the signalling is connected. CI runs the bridges' new tests. Session: claude b0eec5e0-9c0a-43e6-bf1b-01b7e83c523c on jonas-macbook-pro Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 5 +- AGENTS.md | 3 +- README.md | 15 +- Sources/Live/Bridge.swift | 5 +- Sources/VoiceMeetMac/DeviceEvents.swift | 12 ++ Sources/VoiceMeetMac/DeviceLive.swift | 14 +- bridge/device-events.mjs | 141 +++++++++++----- bridge/device-events.test.mjs | 96 +++++++++++ bridge/device-live.mjs | 216 ++++++++++++++++-------- bridge/device-live.test.mjs | 75 ++++++++ bridge/fake-mind.test-helper.mjs | 81 +++++++++ bridge/mind-connection.mjs | 111 ++++++++++++ bridge/mind-connection.test.mjs | 76 +++++++++ 13 files changed, 722 insertions(+), 128 deletions(-) create mode 100644 bridge/device-events.test.mjs create mode 100644 bridge/device-live.test.mjs create mode 100644 bridge/fake-mind.test-helper.mjs create mode 100644 bridge/mind-connection.mjs create mode 100644 bridge/mind-connection.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b64b92..a8784a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ -# Every pull request and every push to main: the app builds, its tests pass, and the bridge's -# scripts parse. +# Every pull request and every push to main: the app builds, its tests pass, the bridge's +# scripts parse, and the bridges' tests pass. name: CI on: pull_request: @@ -16,3 +16,4 @@ jobs: - run: swift build - run: swift test - run: for file in bridge/*.mjs bridge/*.ts cli.mjs; do node --check "$file"; done + - run: node --test 'bridge/*.test.mjs' diff --git a/AGENTS.md b/AGENTS.md index c94a035..4c85477 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,8 @@ how it is installed, the files in a mind it writes, and how to develop it. - Swift in `Sources/`, the Node bridge in `bridge/` (it imports only Node and the published `iterate` package), the installer in `cli.mjs`. -- Check before a pull request: `swift build`, `swift test`, and `node --check` on each changed script. +- Check before a pull request: `swift build`, `swift test`, `node --test 'bridge/*.test.mjs'`, and + `node --check` on each changed script. - Look at a view without installing: the `--render-*` flags (README, "Developing"). - Install a build on this Mac with `node cli.mjs update`. Only one session reinstalls at a time. - Every push to `main` is published to npm and reaches installed Macs within the hour. diff --git a/README.md b/README.md index 0a8b890..54b4586 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,11 @@ An act that was refused (the thing is off, or you said no) is no act and shows i A lent Mac has a context of its own in the mind, `/devices/`, with the name of its lend (`itx.`). The app adds events there, through `bridge/device-events.mjs`, as you (the iterate -CLI's own sign-in; the Mac's key lends and does nothing else): +CLI's own sign-in; the Mac's key lends and does nothing else). When that connection ends (a +session's sign-in lasts 4 hours, a network goes), the bridge connects again +(`bridge/mind-connection.mjs`: 1 s, then twice as long, at most 60 s); the events wait meanwhile +(at most 2,000: the oldest moves and keys go first), go up in order, and the app says its presence +again: | Event | When | Payload | | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | @@ -354,7 +358,9 @@ a binary target of `Package.swift`, carried in the bundle's `Contents/Frameworks the sink and one peer connection per watcher; the signalling's rules are `LiveWatchers` in `VoiceMeetCore`, tested on their own. `bridge/device-live.mjs` connects to the mind as you, subscribes to `device/live-signal`, appends the Mac's signals in order, and asks the facet -`record` for the ICE servers (STUN alone when it has none). +`record` for the ICE servers (STUN alone when it has none). It connects again when its connection +ends, and once a minute it checks that its subscription still hears (an ephemeral +`device/live-ping` that only it hears); watches that run go on through a reconnect, peer to peer. ## Capture: show an agent what you mean @@ -457,8 +463,8 @@ Notes from the first runs (macOS 15.7.4, a MacBook Pro): This repository is the app's home: `git clone https://github.com/iterate/mac-app`, then build and install from the checkout with `node cli.mjs update` (or `./run-on-mac.sh`). A pull request is -checked by `.github/workflows/ci.yml` on a macOS runner: `swift build`, `swift test`, and every -script of the bridge parses. Every push to `main` publishes a version to npm +checked by `.github/workflows/ci.yml` on a macOS runner: `swift build`, `swift test`, every +script of the bridge parses, and the bridges' tests pass (`node --test 'bridge/*.test.mjs'`). Every push to `main` publishes a version to npm (`.github/workflows/release.yml`), and a Mac that installed from npm updates itself within the hour. ```sh @@ -466,6 +472,7 @@ script of the bridge parses. Every push to `main` publishes a version to npm ./run-on-mac.sh --build-only # build and sign .build/iterate.app only ./run-on-mac.sh --logs # follow the app's log swift test # the state machines' tests (needs Xcode): microphone, camera, lend, banner, the record, the mark, record.set's rule, live +node --test 'bridge/*.test.mjs' # the bridges' tests: a fake mind that ends sessions (1008) and subscriptions npx . # the installer from this folder npx . init /tmp/mac --no-install # a copy, without building or installing it ``` diff --git a/Sources/Live/Bridge.swift b/Sources/Live/Bridge.swift index 25cfca0..7001602 100644 --- a/Sources/Live/Bridge.swift +++ b/Sources/Live/Bridge.swift @@ -5,7 +5,8 @@ import VoiceMeetCore /// back, as JSON lines. /// /// - from the bridge: `{ signal: { from, want | description | candidate | bye }, who }`, -/// `{ iceServers: [...] }`, and `{ failed }` for the log; +/// `{ iceServers: [...] }`, `{ connected: true | false, failed? }` (the signalling is up or +/// down), and `{ failed }` for the log; /// - to the bridge: `{ signal: { to, description, tracks } | { to, candidate } | { to, bye, reason } }`, /// `{ started: { watcher, who, tracks } }` and `{ ended: { watcher, who, seconds, why } }`. public final class LiveBridge: @unchecked Sendable { @@ -13,7 +14,7 @@ public final class LiveBridge: @unchecked Sendable { private let input = Pipe() private let lock = NSLock() private var pending = "" - /// A line of the bridge for the log: `{ failed }`, `{ connected }`. + /// A line of the bridge for the log: `{ failed }`, and `{ connected: true | false }`. public var said: ([String: Any]) -> Void = { _ in } public var exited: (Int32) -> Void = { _ in } diff --git a/Sources/VoiceMeetMac/DeviceEvents.swift b/Sources/VoiceMeetMac/DeviceEvents.swift index 90ad7ee..1e2108e 100644 --- a/Sources/VoiceMeetMac/DeviceEvents.swift +++ b/Sources/VoiceMeetMac/DeviceEvents.swift @@ -52,6 +52,7 @@ final class DeviceEvents: ObservableObject { private var input: FileHandle? private var publishesTo: Destination? private var pending = "" + private var wasDown = false private var timer: Timer? private var monitors: [Any] = [] private var lastMoveAt = Date.distantPast @@ -225,6 +226,16 @@ final class DeviceEvents: ObservableObject { if failure != why { log.error("not published: \(why)") } failure = why } + // the bridge connected again: the mind hears the presence of now, not of before + if let up = said["connected"] as? Bool { + if up, wasDown { + log.info("connected again") + failure = nil + presence = nil + tick() + } + wasDown = !up + } } } @@ -234,6 +245,7 @@ final class DeviceEvents: ObservableObject { publisher = nil publishesTo = nil presence = nil + wasDown = false } } diff --git a/Sources/VoiceMeetMac/DeviceLive.swift b/Sources/VoiceMeetMac/DeviceLive.swift index e581f69..f768a5b 100644 --- a/Sources/VoiceMeetMac/DeviceLive.swift +++ b/Sources/VoiceMeetMac/DeviceLive.swift @@ -45,6 +45,8 @@ final class DeviceLive: ObservableObject { private var bridge: LiveBridge? private var bridgeTo: Destination? private var bridgeEndedAt = Date.distantPast + /// The bridge's word: its connection to the mind is up (`{ connected }`). + private var isSignalling = false private var acts: [String: Int] = [:] private var timer: Timer? @@ -100,7 +102,7 @@ final class DeviceLive: ObservableObject { func state() -> [String: Any] { [ "watchers": watchers.map { ["watcher": $0.id, "who": $0.who, "tracks": $0.tracks, "since": Int($0.since.timeIntervalSince1970 * 1000)] }, - "signalling": bridge != nil, + "signalling": bridge != nil && isSignalling, ] } @@ -110,20 +112,24 @@ final class DeviceLive: ObservableObject { if destination != bridgeTo, let old = bridge { peer.endAll(why: "the Mac is no longer lent to this mind") old.stop() - (bridge, bridgeTo, bridgeEndedAt) = (nil, nil, Date()) + (bridge, bridgeTo, bridgeEndedAt, isSignalling) = (nil, nil, Date(), false) } guard let destination, bridge == nil, Date().timeIntervalSince(bridgeEndedAt) > 10 else { return } let made = LiveBridge( node: destination.node, script: destination.script, arguments: [destination.name, "--iterate", destination.iterateBin, "--mind", destination.mind], peer: peer) - made.said = { line in AppLog("live").info("bridge: \(line)") } + made.said = { [weak self] line in + AppLog("live").info("bridge: \(line)") + guard let up = line["connected"] as? Bool else { return } + DispatchQueue.main.async { MainActor.assumeIsolated { if self?.bridge === made { self?.isSignalling = up } } } + } made.exited = { [weak self] status in DispatchQueue.main.async { MainActor.assumeIsolated { guard let self, self.bridge === made else { return } self.log.info("the bridge ended (status \(status))") self.peer.endAll(why: "the signalling stopped") - (self.bridge, self.bridgeTo, self.bridgeEndedAt) = (nil, nil, Date()) + (self.bridge, self.bridgeTo, self.bridgeEndedAt, self.isSignalling) = (nil, nil, Date(), false) } } } diff --git a/bridge/device-events.mjs b/bridge/device-events.mjs index 672ebf0..6485e46 100644 --- a/bridge/device-events.mjs +++ b/bridge/device-events.mjs @@ -6,59 +6,110 @@ // // The app (Sources/VoiceMeetMac/DeviceEvents.swift) starts it and writes one JSON line for each // event on stdin: { type, payload, ephemeral? }. An ephemeral event reaches whoever listens now and -// is in no log. It connects to the mind once, as the person (the iterate CLI's own sign-in: the -// Mac's key lends and does nothing else), and appends what came in every fifth of a second, in one -// call. stdout is JSON lines for the app's log: { sent } and { failed }. +// is in no log. It connects to the mind as the person (the iterate CLI's own sign-in: the Mac's +// key lends and does nothing else), connects again when that connection ends +// (mind-connection.mjs), and appends what came in every fifth of a second, in one call, in order. +// While the connection is down the events wait; a batch that failed waits again at the front. +// stdout is JSON lines for the app: { sent }, { failed, dropped? }, and { connected } from +// mind-connection.mjs: after `{ connected: true }` the app says its presence again. +import { realpathSync } from "node:fs"; import { dirname, join } from "node:path"; import { createInterface } from "node:readline"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; +import { isConnectionError, lastingConnection } from "./mind-connection.mjs"; -const { positionals, values } = parseArgs({ - allowPositionals: true, - options: { iterate: { type: "string" }, mind: { type: "string" } }, -}); -const name = positionals[0]; -if (!name || !values.iterate || !values.mind) - throw new Error("node device-events.mjs --iterate --mind
"); -const say = (line) => process.stdout.write(`${JSON.stringify(line)}\n`); +/** The input events that go first when too many wait: presence and the rest are kept. */ +const isInput = (event) => /^device\/(key-pressed|mouse-|scrolled)/.test(event?.type); -const { addressOf, connectMind } = await import( - pathToFileURL(join(dirname(dirname(values.iterate)), "dist/node.mjs")).href -); -const device = connectMind({ address: addressOf(values.mind), signIn: false }).then(({ itx }) => - itx.cd(`/devices/${name}`), -); -// a failure is told with the first batch -device.catch(() => {}); +/** The queue and its sender: `add(event)`, `send()` (also every `everyMs`), `end()`. */ +export function deviceEvents({ name, connect, say, sleep, mostWaiting = 2_000, everyMs = 200 }) { + const link = lastingConnection({ + connect, + say, + sleep, + connected: async (link) => (link.device = link.itx.cd(`/devices/${name}`)), + }); + let waiting = []; + let sent = 0; + let refusals = 0; + let sending; -// At most this many wait: a Mac with no network drops the oldest moves and keys, it does not grow. -const mostWaiting = 2_000; -let waiting = []; -let sent = 0; -let isSending = false; + // At most `mostWaiting` wait: a Mac with no network drops its oldest moves and keys, it does + // not grow. + function bound() { + while (waiting.length > mostWaiting) { + const input = waiting.findIndex(isInput); + waiting.splice(input === -1 ? 0 : input, 1); + } + } -async function send() { - if (isSending || waiting.length === 0) return; - isSending = true; - const batch = waiting; - waiting = []; - try { - await (await device).append(...batch); - sent += batch.length; - say({ sent }); - } catch (error) { - say({ failed: String(error?.message ?? error).slice(0, 300), dropped: batch.length }); + async function sendOnce() { + if (waiting.length === 0) return; + const connection = await link.now(); + const batch = waiting; + waiting = []; + try { + await connection.device.append(...batch); + sent += batch.length; + refusals = 0; + say({ sent }); + } catch (error) { + const failed = String(error?.message ?? error).slice(0, 300); + const isBroken = isConnectionError(error); + // the mind refused the batch three times: it is dropped, so it holds back nothing + if (!isBroken && ++refusals >= 3) { + refusals = 0; + return say({ failed, dropped: batch.length }); + } + waiting = [...batch, ...waiting]; + bound(); + if (isBroken) link.broke(connection, error); + else say({ failed }); + } } - isSending = false; + const send = () => (sending ??= sendOnce().finally(() => (sending = undefined))); + const timer = setInterval(() => send().catch(() => {}), everyMs); + + return { + add(event) { + waiting.push(event); + bound(); + }, + send, + waiting: () => waiting, + /** What waits goes, for at most `ms`; then the connection ends. */ + async end(ms = 5000) { + clearInterval(timer); + const flushed = (async () => { + while (waiting.length) await send(); + })(); + await Promise.race([flushed.catch(() => {}), new Promise((r) => setTimeout(r, ms).unref())]); + link.stop(); + }, + }; } -setInterval(send, 200); -for await (const line of createInterface({ input: process.stdin })) { - if (!line) continue; - waiting.push(JSON.parse(line)); - if (waiting.length > mostWaiting) waiting.splice(0, waiting.length - mostWaiting); +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { iterate: { type: "string" }, mind: { type: "string" } }, + }); + const name = positionals[0]; + if (!name || !values.iterate || !values.mind) + throw new Error("node device-events.mjs --iterate --mind
"); + const { addressOf, connectMind } = await import( + pathToFileURL(join(dirname(dirname(values.iterate)), "dist/node.mjs")).href + ); + const events = deviceEvents({ + name, + connect: () => connectMind({ address: addressOf(values.mind), signIn: false }), + say: (line) => process.stdout.write(`${JSON.stringify(line)}\n`), + }); + for await (const line of createInterface({ input: process.stdin })) { + if (line) events.add(JSON.parse(line)); + } + // stdin closed: the app is gone, or it turned the events off + await events.end(); + process.exit(0); } -// stdin closed: the app is gone, or it turned the events off -await send(); -process.exit(0); diff --git a/bridge/device-events.test.mjs b/bridge/device-events.test.mjs new file mode 100644 index 0000000..256af7d --- /dev/null +++ b/bridge/device-events.test.mjs @@ -0,0 +1,96 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { deviceEvents } from "./device-events.mjs"; +import { SIGN_IN_ENDED, fakeMind, until } from "./fake-mind.test-helper.mjs"; + +const start = (mind, more = {}) => { + const said = []; + const events = deviceEvents({ + name: "test", + connect: mind.connect, + sleep: mind.sleep, + say: (line) => said.push(line), + everyMs: 3_600_000, + ...more, + }); + return { events, said }; +}; +const presence = (n) => ({ type: "device/presence-changed", payload: { n } }); +const move = (n) => ({ type: "device/mouse-moved", ephemeral: true, payload: { n } }); + +test("events wait while the connection is down and go up after it, in order, none lost", async () => { + const mind = fakeMind(); + const { events, said } = start(mind); + events.add(presence(1)); + await events.send(); + assert.deepEqual(mind.appended.map((e) => e.payload.n), [1]); + + // the mind ends the session; events come in meanwhile, and the first batch fails with 1008 + mind.connections[0].end(); + mind.refuseConnects = 3; + for (let n = 2; n <= 6; n++) events.add(move(n)); + const sending = events.send(); + events.add(presence(7)); + await sending; + await events.send(); + assert.deepEqual(mind.appended.map((e) => e.payload.n), [1, 2, 3, 4, 5, 6, 7]); + assert.equal(mind.connections.length, 2); + assert.deepEqual(said.filter((l) => "connected" in l).map((l) => l.connected), [true, false, false, false, false, true]); + await events.end(); +}); + +test("a batch that failed because the connection broke is tried again on the next one", async () => { + const mind = fakeMind(); + const { events } = start(mind); + events.add(presence(1)); + events.add(move(2)); + mind.failNextAppend = new Error(`Peer closed WebSocket: ${SIGN_IN_ENDED}`); + await events.send(); + assert.deepEqual(mind.appended, []); + await events.send(); + assert.deepEqual(mind.appended.map((e) => e.payload.n), [1, 2]); + assert.equal(mind.connections.length, 2); + await events.end(); +}); + +test("over the bound, the oldest moves and keys go first; presence is kept", async () => { + const mind = fakeMind(); + mind.refuseConnects = 1_000; + const { events } = start(mind, { mostWaiting: 4 }); + events.add(presence(1)); + events.add(move(2)); + events.add(move(3)); + events.add(presence(4)); + events.add(move(5)); + events.add(move(6)); + assert.deepEqual(events.waiting().map((e) => e.payload.n), [1, 4, 5, 6]); + events.add(presence(7)); + events.add(presence(8)); + events.add(presence(9)); + assert.deepEqual(events.waiting().map((e) => e.payload.n), [4, 7, 8, 9]); + await events.end(10); +}); + +test("a batch the mind refuses three times is dropped, and holds back nothing", async () => { + const mind = fakeMind(); + const { events, said } = start(mind); + events.add(presence(1)); + for (let i = 0; i < 3; i++) { + mind.failNextAppend = Object.assign(new Error("not valid"), { code: "INVALID_INPUT" }); + await events.send(); + } + assert.deepEqual(said.at(-1), { failed: "not valid", dropped: 1 }); + events.add(presence(2)); + await events.send(); + assert.deepEqual(mind.appended.map((e) => e.payload.n), [2]); + assert.equal(mind.connections.length, 1, "a refusal is no broken connection"); + await events.end(); +}); + +test("the end sends what waits", async () => { + const mind = fakeMind(); + const { events } = start(mind); + events.add(presence(1)); + await events.end(); + await until(() => mind.appended.length === 1); +}); diff --git a/bridge/device-live.mjs b/bridge/device-live.mjs index 29283a6..4cb919d 100644 --- a/bridge/device-live.mjs +++ b/bridge/device-live.mjs @@ -6,57 +6,34 @@ // node device-live.mjs --iterate --mind
// // The app (Sources/VoiceMeetMac/DeviceLive.swift, through Sources/Live/Bridge.swift) starts it -// while the Mac is lent, and closes stdin to end it. It connects to the mind once, as the person -// (the iterate CLI's own sign-in: the Mac's key lends and does nothing else). +// while the Mac is lent, and closes stdin to end it. It connects to the mind as the person (the +// iterate CLI's own sign-in: the Mac's key lends and does nothing else), and connects again when +// that connection ends (mind-connection.mjs). On each connection it subscribes and asks for the ICE +// servers. A subscription can end on the mind's side with no word: once a minute the bridge +// appends an ephemeral `device/live-ping` that only its own subscription hears, and a ping not +// heard by the next one ends the connection, which connects and subscribes again. Watches that +// run go on through it: they are peer to peer. // -// - stdout: `{ iceServers }` first and again at half their lifetime: what `iceServers()` of the -// facet `record` on /devices/ answers (STUN, and TURN when the mind has it); without that -// facet the app keeps STUN alone. Then each `device/live-signal` whose `to` is "device", as -// `{ signal: { from, want | description | candidate | bye }, who }`: `who` is the event's -// source, as an email or a short name. `{ failed }` lines are for the app's log. +// - stdout: `{ iceServers }` on each connection and again at half their lifetime: what +// `iceServers()` of the facet `record` on /devices/ answers (STUN, and TURN when the mind +// has it); without that facet the app keeps STUN alone. Then each `device/live-signal` whose +// `to` is "device", as `{ signal: { from, want | description | candidate | bye }, who }`: `who` +// is the event's source, as an email or a short name. `{ connected: true }` and +// `{ connected: false, failed }` say whether the signalling is up; `{ failed }` lines are for +// the app's log. // - stdin: `{ signal: { to, … } }` becomes an ephemeral `device/live-signal { from: "device", to, … }`, // appended in order (an offer before its candidates). `{ started }` and `{ ended }` become the // kept events `device/live-watch-started { watcher, who, tracks }` and // `device/live-watch-ended { watcher, who, seconds, why }`. +import { realpathSync } from "node:fs"; import { dirname, join } from "node:path"; import { createInterface } from "node:readline"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; +import { isConnectionError, lastingConnection } from "./mind-connection.mjs"; -const { positionals, values } = parseArgs({ - allowPositionals: true, - options: { iterate: { type: "string" }, mind: { type: "string" } }, -}); -const name = positionals[0]; -if (!name || !values.iterate || !values.mind) - throw new Error("node device-live.mjs --iterate --mind
"); -const say = (line) => process.stdout.write(`${JSON.stringify(line)}\n`); -const failed = (what, error) => say({ failed: `${what}: ${String(error?.message ?? error).slice(0, 300)}` }); const SIGNAL = "device/live-signal"; - -const { addressOf, connectMind } = await import( - pathToFileURL(join(dirname(dirname(values.iterate)), "dist/node.mjs")).href -); -const { itx } = await connectMind({ address: addressOf(values.mind), signIn: false }); -const device = itx.cd(`/devices/${name}`); - -// The addresses a connection may use, before any watcher; again at half their lifetime. -async function iceServers() { - let ttl = 3600; - try { - const answer = await Promise.race([ - device.facets.get("record").iceServers(), - new Promise((_, reject) => setTimeout(() => reject(new Error("no answer in 5 s")), 5000)), - ]); - const servers = JSON.parse(JSON.stringify(answer?.iceServers ?? [])); - if (servers.length) say({ iceServers: servers }); - ttl = answer?.ttlSeconds ?? ttl; - } catch (error) { - failed("ICE servers, STUN alone", error); - } - setTimeout(iceServers, Math.max(60, ttl / 2) * 1000).unref(); -} -await iceServers(); +const PING = "device/live-ping"; /** Who asked: the principal's email, else its actor, else the origin. */ function whoOf(source) { @@ -64,36 +41,135 @@ function whoOf(source) { return String(principal?.email ?? principal?.actor ?? source?.origin ?? "someone").slice(0, 100); } -// In: what the watchers say to the device. -const subscription = await device.subscribe({ - name: `device-live-${process.pid}-${Date.now().toString(36)}`, - consumes: [SIGNAL], - target: (batch) => { - for (const event of JSON.parse(JSON.stringify(batch))) { - const payload = event?.payload; - if (event?.type !== SIGNAL || payload?.to !== "device" || !payload.from) continue; - const { to, at, ...signal } = payload; - say({ signal, who: whoOf(event.source) }); +const within = (promise, ms) => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`no answer in ${ms / 1000} s`)), ms).unref()), + ]); + +/** The signalling: `say` hears the watchers, `append(line)` takes a line of the app. */ +export function deviceLive({ name, connect, say, sleep, pingEveryMs = 60_000 }) { + const failed = (what, error) => say({ failed: `${what}: ${String(error?.message ?? error).slice(0, 300)}` }); + const me = `device-live-${process.pid}-${Date.now().toString(36)}`; + let iceTimer; + let pings = 0; + let subscriptions = 0; + + // The addresses a connection may use, before any watcher; again at half their lifetime. + async function iceServers(device) { + clearTimeout(iceTimer); + let ttl = 3600; + try { + const answer = await within(device.facets.get("record").iceServers(), 5000); + const servers = JSON.parse(JSON.stringify(answer?.iceServers ?? [])); + if (servers.length) say({ iceServers: servers }); + ttl = answer?.ttlSeconds ?? ttl; + } catch (error) { + failed("ICE servers, STUN alone", error); + } + iceTimer = setTimeout( + () => link.now().then((connection) => iceServers(connection.device)), + Math.max(60, ttl / 2) * 1000, + ); + iceTimer.unref?.(); + } + + const link = lastingConnection({ + connect, + say, + sleep, + async connected(connection) { + const device = (connection.device = connection.itx.cd(`/devices/${name}`)); + connection.subscription = await device.subscribe({ + name: `${me}-${++subscriptions}`, + consumes: [SIGNAL, PING], + target: (batch) => { + for (const event of JSON.parse(JSON.stringify(batch))) { + const payload = event?.payload; + if (event?.type === PING && payload?.from === me) connection.heard = payload.ping; + if (event?.type !== SIGNAL || payload?.to !== "device" || !payload.from) continue; + const { to, at, ...signal } = payload; + say({ signal, who: whoOf(event.source) }); + } + }, + }); + await iceServers(device); + }, + }); + + // A ping that its own subscription did not hear by the next one: the subscription is gone. + async function ping() { + const connection = link.current(); + if (!connection) return; + if (connection.pinged !== connection.heard) + return link.broke(connection, new Error("the subscription heard nothing in a minute")); + connection.pinged = ++pings; + try { + await within(connection.device.append({ type: PING, ephemeral: true, payload: { from: me, ping: pings } }), 20_000); + } catch (error) { + link.broke(connection, error); + } + } + const pinger = setInterval(() => ping().catch(() => {}), pingEveryMs); + + // Out: what the device says, one append at a time, in order; on up to three connections. + let chain = Promise.resolve(); + async function appendOne(event) { + for (let tries = 1; tries <= 3; tries++) { + const connection = await link.now(); + try { + return await connection.device.append(event); + } catch (error) { + failed(event.type, error); + if (!isConnectionError(error)) return; + link.broke(connection, error); + } } - }, -}); + } + const append = (event) => (chain = chain.then(() => appendOne(event)).catch(() => {})); + + return { + ping, + /** A line of the app. */ + take(line) { + if (line.signal) append({ type: SIGNAL, ephemeral: true, payload: { from: "device", ...line.signal } }); + for (const kind of ["started", "ended"]) { + if (!line[kind]) continue; + const type = `device/live-watch-${kind}`; + append({ type, idempotencyKey: `${type}/${line[kind].watcher}`, payload: line[kind] }); + } + return chain; + }, + /** The last byes go out first, for at most `ms`. */ + async end(ms = 5000) { + clearInterval(pinger); + clearTimeout(iceTimer); + await within(chain, ms).catch(() => {}); + link.stop(); + }, + }; +} -// Out: what the device says, one append at a time, in order. -let chain = Promise.resolve(); -const append = (event) => { - chain = chain.then(() => device.append(event)).catch((error) => failed(event.type, error)); -}; -for await (const text of createInterface({ input: process.stdin })) { - if (!text) continue; - const line = JSON.parse(text); - if (line.signal) append({ type: SIGNAL, ephemeral: true, payload: { from: "device", ...line.signal } }); - for (const kind of ["started", "ended"]) { - if (!line[kind]) continue; - const type = `device/live-watch-${kind}`; - append({ type, idempotencyKey: `${type}/${line[kind].watcher}`, payload: line[kind] }); +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { iterate: { type: "string" }, mind: { type: "string" } }, + }); + const name = positionals[0]; + if (!name || !values.iterate || !values.mind) + throw new Error("node device-live.mjs --iterate --mind
"); + const { addressOf, connectMind } = await import( + pathToFileURL(join(dirname(dirname(values.iterate)), "dist/node.mjs")).href + ); + const live = deviceLive({ + name, + connect: () => connectMind({ address: addressOf(values.mind), signIn: false }), + say: (line) => process.stdout.write(`${JSON.stringify(line)}\n`), + }); + for await (const text of createInterface({ input: process.stdin })) { + if (text) live.take(JSON.parse(text)); } + // stdin closed: the app is gone, or the Mac is no longer lent + await live.end(); + process.exit(0); } -// stdin closed: the app is gone, or the Mac is no longer lent. The last byes go out first. -await chain; -subscription?.[Symbol.dispose]?.(); -process.exit(0); diff --git a/bridge/device-live.test.mjs b/bridge/device-live.test.mjs new file mode 100644 index 0000000..3f2b5df --- /dev/null +++ b/bridge/device-live.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { deviceLive } from "./device-live.mjs"; +import { fakeMind, until } from "./fake-mind.test-helper.mjs"; + +const start = (mind) => { + const said = []; + const live = deviceLive({ + name: "test", + connect: mind.connect, + sleep: mind.sleep, + say: (line) => said.push(line), + pingEveryMs: 3_600_000, + }); + return { live, said }; +}; +const want = (from) => ({ + type: "device/live-signal", + payload: { from, to: "device", want: true }, + source: { principal: { email: "watcher@example.com" } }, +}); +const heard = (said) => said.filter((l) => l.signal).map((l) => l.signal.from); +const alive = (mind) => mind.subscriptions.filter((s) => s.alive && s.connection.isOpen); + +test("after each new connection it subscribes again, asks for the ICE servers, and hears", async () => { + const mind = fakeMind(); + const { live, said } = start(mind); + await until(() => alive(mind).length === 1); + mind.deliver([want("a")]); + assert.deepEqual(heard(said), ["a"]); + + mind.connections[0].end(); + await until(() => alive(mind).length === 1 && mind.connections.length === 2); + mind.deliver([want("b")]); + assert.deepEqual(heard(said), ["a", "b"]); + assert.equal(said.filter((l) => l.iceServers).length, 2); + assert.deepEqual(said.filter((l) => "connected" in l).map((l) => l.connected), [true, false, true]); + await live.end(); +}); + +test("a subscription that ends with no word is found by the ping and made again", async () => { + const mind = fakeMind(); + const { live, said } = start(mind); + await until(() => alive(mind).length === 1); + await live.ping(); + await live.ping(); + assert.equal(mind.connections.length, 1, "a heard ping changes nothing"); + + mind.subscriptions[0].alive = false; + await live.ping(); + await live.ping(); + await until(() => alive(mind).length === 1 && mind.connections.length === 2); + mind.deliver([want("c")]); + assert.deepEqual(heard(said), ["c"]); + assert.match(said.find((l) => l.connected === false).failed, /heard nothing/); + await live.end(); +}); + +test("what the device says goes out in order, and waits for the connection that comes back", async () => { + const mind = fakeMind(); + const { live } = start(mind); + await until(() => alive(mind).length === 1); + mind.connections[0].end(); + live.take({ signal: { to: "w", description: { type: "offer" } } }); + live.take({ signal: { to: "w", candidate: { candidate: "c1" } } }); + await live.take({ started: { watcher: "w", who: "x", tracks: [] } }); + const sent = mind.appended.filter((e) => e.type !== "device/live-ping"); + assert.deepEqual( + sent.map((e) => e.type), + ["device/live-signal", "device/live-signal", "device/live-watch-started"], + ); + assert.equal(sent[0].payload.description.type, "offer"); + assert.equal(sent[2].idempotencyKey, "device/live-watch-started/w"); + await live.end(); +}); diff --git a/bridge/fake-mind.test-helper.mjs b/bridge/fake-mind.test-helper.mjs new file mode 100644 index 0000000..fc2cabc --- /dev/null +++ b/bridge/fake-mind.test-helper.mjs @@ -0,0 +1,81 @@ +// A fake mind for the bridges' tests: connections that end as the mind ends them (1008), that +// fail to connect, subscriptions that end with no word, appends that fail. +export const SIGN_IN_ENDED = "1008 the sign-in for this session ended: authenticate again"; + +export function fakeMind() { + const mind = { + connections: [], + subscriptions: [], + appended: [], + refuseConnects: 0, + failNextAppend: undefined, + waits: [], + clock: 0, + }; + mind.sleep = (ms) => { + mind.waits.push(ms); + return new Promise((resolve) => setImmediate(resolve)); + }; + mind.now = () => mind.clock; + mind.deliver = (events, from) => { + for (const s of mind.subscriptions) + if (s.alive && s.connection.isOpen && (!from || s.connection === from)) + s.target(events.filter((event) => s.consumes.includes(event.type))); + }; + mind.connect = async () => { + if (mind.refuseConnects > 0) { + mind.refuseConnects--; + throw new Error("connect ECONNREFUSED"); + } + let close; + const connection = { + isOpen: true, + closed: new Promise((resolve) => (close = resolve)), + /** The mind ends the session: the socket closes with 1008. */ + end() { + connection.isOpen = false; + close({ code: 1008, reason: SIGN_IN_ENDED.slice(5) }); + }, + [Symbol.dispose]() { + connection.isOpen = false; + close({ code: 3000, reason: "disposed" }); + }, + }; + connection.itx = { + cd: (path) => ({ + path, + async append(...events) { + if (!connection.isOpen) throw new Error(`Peer closed WebSocket: ${SIGN_IN_ENDED}`); + if (mind.failNextAppend) { + const error = mind.failNextAppend; + mind.failNextAppend = undefined; + throw error; + } + mind.appended.push(...events); + mind.deliver(events, connection); + return events; + }, + async subscribe({ name, consumes, target }) { + const subscription = { name, consumes, target, connection, alive: true }; + mind.subscriptions.push(subscription); + return { [Symbol.dispose]: () => (subscription.alive = false) }; + }, + facets: { + get: () => ({ iceServers: async () => ({ iceServers: [{ urls: "stun:test" }], ttlSeconds: 3600 }) }), + }, + }), + }; + mind.connections.push(connection); + return connection; + }; + return mind; +} + +/** Waits, a turn of the event loop at a time, until `check()` is true. */ +export async function until(check, turns = 500) { + for (let i = 0; i < turns; i++) { + if (check()) return; + await new Promise((resolve) => setImmediate(resolve)); + } + throw new Error(`never true: ${check}`); +} diff --git a/bridge/mind-connection.mjs b/bridge/mind-connection.mjs new file mode 100644 index 0000000..17e39dc --- /dev/null +++ b/bridge/mind-connection.mjs @@ -0,0 +1,111 @@ +// mind-connection.mjs: a connection to the mind that lasts, for the bridges that hold one for as +// long as the Mac is lent (device-live.mjs, device-events.mjs). +// +// `connect()` makes one connection (`connectMind`: `{ itx, closed, [Symbol.dispose] }`), and each +// new connection reads the stored sign-in again, so a session that ended after its 4 hours comes +// back with a renewed one. `connected(link)` runs on each new connection before anyone gets it +// (subscribe, ask for the ICE servers); when it throws, the connection is no good. A connection +// ends when its `closed` resolves (the socket closed: 1008 from the mind, or the heartbeat found +// no answer), or when a user says so with `broke(link)` (a call failed with a closed connection, a +// subscription that hears nothing). Then it connects again after a wait of 1 s, twice as long +// after each failure, at most 60 s; the wait starts at 1 s again after a connection that lasted a +// minute. It never gives up. stdout hears `{ connected: true }` and `{ connected: false, failed }`. + +const describe = (error) => String(error?.message ?? error?.reason ?? error).slice(0, 300); +const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Whether a failed call says the connection is gone, not that the mind refused the call. */ +export const isConnectionError = (error) => + /\b1008\b|closed|disposed|shut down|broken|WebSocket|no answer|timed out|ECONN|ENOTFOUND|EAI_AGAIN|socket/i.test( + describe(error), + ); + +export function lastingConnection({ + connect, + connected = async () => {}, + say = () => {}, + sleep = pause, + now = Date.now, + giveUpAfterMs = 60_000, +}) { + let current; + let ended = false; + let waiters = []; + let endCurrent = () => {}; + + (async () => { + let wait = 1000; + while (!ended) { + let link; + let upAt; + let timer; + try { + const attempt = (async () => { + link = await connect(); + await connected(link); + return link; + })(); + // a connection that hangs is given up; one that comes after that is closed + await Promise.race([ + attempt, + new Promise((_, reject) => { + timer = setTimeout(() => { + attempt.then(dispose, () => {}); + reject(new Error(`no connection in ${giveUpAfterMs / 1000} s`)); + }, giveUpAfterMs); + }), + ]).finally(() => clearTimeout(timer)); + if (ended) { + dispose(link); + break; + } + const broken = new Promise((resolve) => (endCurrent = resolve)); + upAt = now(); + current = link; + say({ connected: true }); + for (const waiter of waiters.splice(0)) waiter.resolve(link); + const why = await Promise.race([link.closed, broken]); + current = undefined; + // `closed` answers { code, reason }: 1008 is a session whose sign-in ended + say({ connected: false, failed: why?.code ? `${why.code} ${why.reason}` : describe(why) }); + } catch (error) { + current = undefined; + say({ connected: false, failed: describe(error) }); + } + dispose(link); + if (ended) break; + if (upAt !== undefined && now() - upAt >= 60_000) wait = 1000; + await sleep(wait); + wait = Math.min(60_000, wait * 2); + } + })(); + + return { + /** The connection that is up, or none. */ + current: () => current, + /** The connection that is up, or the next one. */ + now: () => + current + ? Promise.resolve(current) + : ended + ? Promise.reject(new Error("ended")) + : new Promise((resolve, reject) => waiters.push({ resolve, reject })), + /** `link` is no good (a call on it failed with a closed connection): connect again. */ + broke(link, why) { + if (link && link === current) endCurrent(why ?? "broken"); + }, + stop() { + ended = true; + for (const waiter of waiters.splice(0)) waiter.reject(new Error("ended")); + endCurrent("stopped"); + dispose(current); + current = undefined; + }, + }; +} + +function dispose(link) { + try { + link?.[Symbol.dispose]?.(); + } catch {} +} diff --git a/bridge/mind-connection.test.mjs b/bridge/mind-connection.test.mjs new file mode 100644 index 0000000..64e63d5 --- /dev/null +++ b/bridge/mind-connection.test.mjs @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { SIGN_IN_ENDED, fakeMind, until } from "./fake-mind.test-helper.mjs"; +import { isConnectionError, lastingConnection } from "./mind-connection.mjs"; + +const start = (mind, more = {}) => { + const said = []; + const link = lastingConnection({ + connect: mind.connect, + sleep: mind.sleep, + now: mind.now, + say: (line) => said.push(line), + ...more, + }); + return { link, said }; +}; + +test("a session the mind ends (1008) connects again, and the waits double to 60 s and reset", async () => { + const mind = fakeMind(); + const { link, said } = start(mind); + const first = await link.now(); + assert.equal(link.current(), first); + + mind.refuseConnects = 8; + first.end(); + await until(() => link.current() !== first); + const second = await link.now(); + assert.notEqual(second, first); + assert.deepEqual(mind.waits, [1000, 2000, 4000, 8000, 16000, 32000, 60000, 60000, 60000]); + assert.deepEqual(said[1], { connected: false, failed: SIGN_IN_ENDED }); + assert.deepEqual(said.at(-1), { connected: true }); + + // a connection that lasted a minute: the wait starts at 1 s again + mind.waits.length = 0; + mind.clock += 61_000; + second.end(); + await until(() => link.current() && link.current() !== second); + assert.deepEqual(mind.waits, [1000]); + link.stop(); +}); + +test("broke() ends the connection it names, never a newer one", async () => { + const mind = fakeMind(); + const { link } = start(mind); + const first = await link.now(); + link.broke(first, new Error("Peer closed WebSocket: 1008")); + await until(() => link.current() && link.current() !== first); + const second = link.current(); + link.broke(first); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(link.current(), second); + assert.equal(first.isOpen, false, "the broken one is disposed"); + link.stop(); +}); + +test("a connection whose callback fails is no connection: it is closed, and tried again", async () => { + const mind = fakeMind(); + let calls = 0; + const { link } = start(mind, { + connected: async () => { + if (++calls === 1) throw new Error("subscribe failed"); + }, + }); + const up = await link.now(); + assert.equal(mind.connections.length, 2); + assert.equal(mind.connections[0].isOpen, false); + assert.equal(up, mind.connections[1]); + link.stop(); + await assert.rejects(link.now()); +}); + +test("which failures are a connection's", () => { + assert.ok(isConnectionError(new Error(`Peer closed WebSocket: ${SIGN_IN_ENDED}`))); + assert.ok(isConnectionError(new Error("RPC session was shut down by disposing the main stub"))); + assert.ok(!isConnectionError(Object.assign(new Error("payload is not valid"), { code: "INVALID_INPUT" }))); +});