Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cockpit/App/FleetCockpit/AppModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@ final class AppModel {
}

func openTerminal(alias: String? = nil) {
let host = alias ?? settings.aliasList.first ?? "runner-host"
let host = alias ?? settings.aliasList.first ?? TransportConfig.defaultAliases[0]
let script = "tell application \"Terminal\" to do script \"ssh \(host)\"\ntell application \"Terminal\" to activate"
NSAppleScript(source: script)?.executeAndReturnError(nil)
}
Expand Down
4 changes: 2 additions & 2 deletions cockpit/App/FleetCockpit/AppSettings.swift
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ struct AppSettings: Codable, Equatable {
}

var mode: Mode = .tunnel
var aliases: String = "runner-host, runner-ts"
var remotePort: Int = 7878
var aliases: String = TransportConfig.defaultAliases.joined(separator: ", ")
var remotePort: Int = TransportConfig.defaultRemotePort
var directURL: String = "http://127.0.0.1:7878"
var fixture: String = "live"
var showCounts: Bool = true
Expand Down
2 changes: 1 addition & 1 deletion cockpit/App/FleetCockpitWidgets/FleetWidgets.swift
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ struct FleetProvider: TimelineProvider {

static var sample: SnapshotFile? {
guard let g = try? Fixtures.glance("waiting") else { return nil }
return SnapshotFile(writtenAt: Format.nowMs(), route: "runner-host", connection: "live", verdict: g.verdict, glance: g)
return SnapshotFile(writtenAt: Format.nowMs(), route: TransportConfig.defaultAliases[0], connection: "live", verdict: g.verdict, glance: g)
}
}

Expand Down
8 changes: 8 additions & 0 deletions cockpit/Sources/CockpitCore/SnapshotFile.swift
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ public struct SnapshotFile: Codable, Sendable, Equatable {
self.verdict = verdict; self.glance = glance
}

/// True when the app was talking to a dashboard as it wrote this, not
/// connecting or retrying a route that may no longer exist (an alias for a
/// retired host keeps the snapshot fresh while its glance goes stale).
public var isConnected: Bool {
connection == String(describing: ConnectionState.live)
|| connection == String(describing: ConnectionState.collectorStale)
}

public static var defaultURL: URL {
FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]
.appendingPathComponent("FleetCockpit", isDirectory: true)
Expand Down
10 changes: 9 additions & 1 deletion cockpit/Sources/CockpitCore/Transport.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,15 @@ public enum TransportConfig: Codable, Sendable, Equatable {
case direct(URL)
case fixture(String)

public static let `default` = TransportConfig.tunnel(aliases: ["runner-host", "runner-ts"], remotePort: 7878)
/// The ssh aliases tried, in order, when nothing else is configured: the
/// coordinator's LAN name, then its tailnet name. This is the one place the
/// default route is written down; the app's settings, the CLI's `--via` and
/// the sentinel all start from it. A fleet on another host is a setting
/// (Settings → Connection, or `--via`), not a new build.
public static let defaultAliases = ["ultra", "ultra-ts"]
public static let defaultRemotePort = 7878

public static let `default` = TransportConfig.tunnel(aliases: defaultAliases, remotePort: defaultRemotePort)
}

/// The route in use right now, shown in the popover footer.
Expand Down
22 changes: 14 additions & 8 deletions cockpit/Sources/cockpit/main.swift
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ commands:
options:
--json machine-readable output
--fixture <name> use a bundled fixture instead of the live fleet
--via <alias> ssh alias to tunnel through (repeatable; default: runner-host, runner-ts)
--via <alias> ssh alias to tunnel through (repeatable; default: \(TransportConfig.defaultAliases.joined(separator: ", ")))
--url <url> reach the dashboard directly
--fresh ignore the app's snapshot and connect now
--port <n> the dashboard's port on the host (default 7878)
Expand Down Expand Up @@ -125,18 +125,24 @@ func obtain(_ o: Options) async -> Source {
return Source(glance: g, verdict: g.verdict, route: "fixture: \(name)")
} catch { fail("no fixture named \(name) (try: cockpit fixtures)") }
}
if !o.fresh, o.url == nil, o.via.isEmpty, let snap = try? SnapshotFile.read(), snap.ageMs() < 90_000 {
return Source(glance: snap.glance, verdict: trusted(snap.verdict, snap.glance), route: "app (\(snap.route ?? "?"), \(Format.ago(snap.writtenAt, now: Format.nowMs())))")
// The app's view when it is fresh. If the app is not connected right now,
// try our own route first and keep its view (which may carry an
// out-of-band "host down") as the answer only when ours fails too.
let snap = (!o.fresh && o.url == nil && o.via.isEmpty) ? (try? SnapshotFile.read()).flatMap { $0.ageMs() < 90_000 ? $0 : nil } : nil
func fromApp(_ s: SnapshotFile) -> Source {
Source(glance: s.glance, verdict: trusted(s.verdict, s.glance), route: "app (\(s.route ?? "?"), \(Format.ago(s.writtenAt, now: Format.nowMs())))")
}
if let snap, snap.isConnected { return fromApp(snap) }
let transport: Transport = o.url.map { DirectTransport(url: $0) }
?? TunnelTransport(aliases: o.via.isEmpty ? ["runner-host", "runner-ts"] : o.via)
?? TunnelTransport(aliases: o.via.isEmpty ? defaultAliases : o.via)
do {
let route = try await transport.open()
let client = FleetClient(base: route.baseURL)
let g = try await client.glance()
return Source(glance: g, verdict: trusted(g.verdict, g), route: route.label, client: client, transport: transport)
} catch {
transport.close()
if let snap { return fromApp(snap) }
let v = Verdict(id: "unreachable", tone: .unknown, title: "Dashboard unreachable",
sentence: "Could not reach the dashboard: \(error)", evidence: [], next: nil, rung: nil, open: [])
return Source(glance: nil, verdict: v, route: "none")
Expand Down Expand Up @@ -166,7 +172,7 @@ final class RouteCache: @unchecked Sendable {
}
}

let defaultAliases = ["runner-host", "runner-ts"]
let defaultAliases = TransportConfig.defaultAliases

func transport(_ o: Options) -> Transport {
o.url.map { DirectTransport(url: $0) } ?? TunnelTransport(aliases: o.via.isEmpty ? defaultAliases : o.via)
Expand Down Expand Up @@ -257,7 +263,7 @@ case "status":
src.transport?.close()
exit(exitCode(src.verdict))
case "sentinel":
let aliases = opts.via.isEmpty ? ["runner-host", "runner-ts"] : opts.via
let aliases = opts.via.isEmpty ? defaultAliases : opts.via
let targets = (try? SnapshotFile.read())?.glance.flatMap { SentinelTargets.from($0) }
var probe = SentinelProbe(aliases: aliases)
if let port = opts.flags["port"].flatMap(Int.init) { probe.remotePort = port }
Expand All @@ -271,7 +277,7 @@ case "sentinel":
}
exit(exitCode(v))
case "pair":
let alias = opts.via.first ?? "runner-host"
let alias = opts.via.first ?? defaultAliases[0]
let src = await obtain(Options(command: "status", fresh: true, positional: [], flags: [:]).with(via: opts.via, url: opts.url))
guard let client = src.client, let g = src.glance else { fail("cannot reach the dashboard to pair") }
let account = "cli:" + (g.hosts.first { $0.local == true }?.id ?? alias)
Expand Down Expand Up @@ -430,7 +436,7 @@ case "wait":
fail(e)
}
case "top":
let alias = opts.via.first ?? "runner-host"
let alias = opts.via.first ?? defaultAliases[0]
guard let top = await HostProbe.topCPU(alias: alias) else { fail("could not run ps on \(alias)", code: 3) }
if opts.json {
struct Top: Encodable { let verdict: String; let spotlightCPU: Double; let top: [String] }
Expand Down
13 changes: 13 additions & 0 deletions cockpit/Tests/CockpitCoreTests/CoreTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,19 @@ struct SnapshotFileTests {
let mode = try FileManager.default.attributesOfItem(atPath: url.path)[.posixPermissions] as? Int
#expect(mode == 0o600)
}

/// The app writes its connection state with String(describing:); a
/// snapshot from an app retrying a dead route must not read as connected.
@Test func connectedOnlyWhileTheAppTalksToADashboard() throws {
let g = try Fixtures.glance("live")
func snap(_ c: ConnectionState) -> SnapshotFile {
SnapshotFile(writtenAt: 1000, route: "ultra", connection: String(describing: c), verdict: g.verdict, glance: g)
}
#expect(snap(.live).isConnected)
#expect(snap(.collectorStale).isConnected)
#expect(!snap(.connecting).isConnected)
#expect(!snap(.reconnecting(attempt: 26, error: "No route to the dashboard (tried runner-host, runner-ts)")).isConnected)
}
}

@Suite("transport and client against a stub daemon", .serialized)
Expand Down
2 changes: 1 addition & 1 deletion cockpit/skills/fleet-cockpit/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,5 +91,5 @@ frozen; mention it.
- `cockpit sentinel` checks SSH reachability, GitHub's view of the host's
runners, and another machine.
- `fleet` reports `unreachable`: check `FLEET_URL`, or forward the port with
`ssh -L 7878:localhost:7878 <runner-host>`.
`ssh -L 7878:localhost:7878 <runner host alias>`.
- `cockpit brief` prints a markdown incident brief to paste into a report.
4 changes: 2 additions & 2 deletions dashboard/scripts/settle-stale-jobs.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
//
// Three steps, each restartable:
//
// ssh runner-host "/usr/bin/sqlite3 -json ~/actions-runners/dashboard/fleet.db \
// ssh ultra "/usr/bin/sqlite3 -json ~/actions-runners/dashboard/fleet.db \
// \"$(node scripts/settle-stale-jobs.mjs query)\"" > stale.json
// node scripts/settle-stale-jobs.mjs fetch --in stale.json --out fetched.ndjson
// node scripts/settle-stale-jobs.mjs sql --in fetched.ndjson \
// | ssh runner-host /usr/bin/sqlite3 ~/actions-runners/dashboard/fleet.db
// | ssh ultra /usr/bin/sqlite3 ~/actions-runners/dashboard/fleet.db
//
// fetch skips ids already in --out, so an interrupted run resumes. It paces
// itself under GitHub's primary limit (sleeps to the reset below --floor) and
Expand Down
2 changes: 1 addition & 1 deletion docs/cockpit/agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ the MCP server, the Shortcuts and other local dashboards answer in milliseconds.
```json
{
"writtenAt": 1790781990012,
"route": "runner-host",
"route": "ultra",
"connection": "live",
"verdict": { "id": "waiting", "tone": "ok", "title": "Working", "...": "..." },
"glance": { "schema": 1, "...": "the full GET /api/glance payload" }
Expand Down
20 changes: 10 additions & 10 deletions docs/cockpit/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ trustworthy, in this order:
90 seconds old and you did not pass `--fresh`, `--url` or `--via`. This
answers in milliseconds and opens nothing.
3. **Its own connection**: `--url` directly, otherwise an SSH tunnel through
`--via` aliases (default `runner-host`, then `runner-ts`), exactly as the app
`--via` aliases (default `ultra`, then `ultra-ts`), exactly as the app
would open one.

The human output always ends with the route it used, for example
`via app (runner-host, 12s ago)` or `via runner-ts`.
`via app (ultra, 12s ago)` or `via ultra-ts`.

`why`, `wait`, `run` and `mcp` ask the dashboard's [`/api/v1`](../api.md#version-1-apiv1)
instead, over the same route, and never read the snapshot or a fixture. The
Expand All @@ -37,7 +37,7 @@ than `/api/v1` gets the message to update actions-runners on the host.
|---|---|
| `--json` | Machine-readable output (pretty-printed, sorted keys) |
| `--fixture <name>` | Use a bundled fixture instead of the live fleet (`cockpit fixtures` lists them) |
| `--via <alias>` | SSH alias to tunnel through; repeat for a fallback list. Default `runner-host`, `runner-ts` |
| `--via <alias>` | SSH alias to tunnel through; repeat for a fallback list. Default `ultra`, `ultra-ts` |
| `--url <url>` | Reach the dashboard directly, for example over Tailscale Serve |
| `--fresh` | Ignore the app's snapshot and connect now |
| `--port <n>` | The dashboard's port on the host, for `sentinel`. Other commands tunnel to 7878; use `--url` for a dashboard on another port |
Expand Down Expand Up @@ -65,7 +65,7 @@ $ cockpit status
· 3 jobs held at Set up runner: comet-web, comet-backend, ember-ios
· Waiting does not fix this; freeing disk does
→ Preview cleanup
0 running, 0 queued, 3 held, 54 runners · via app (runner-host, 4s ago)
0 running, 0 queued, 3 held, 54 runners · via app (ultra, 4s ago)
```

The mark is `✔` for ok and info, `▲` warning, `✖` critical, `?` unknown. `→` is
Expand Down Expand Up @@ -288,10 +288,10 @@ runner on another machine, githubstatus.com. Prints the verdict from the

```console
$ cockpit sentinel
✖ runner-host is asleep, off or off the network
✖ ultra is asleep, off or off the network
mini in the same house is online, so power and the network are fine. Only someone there can wake it.
· SSH port closed on every route
· GitHub: 0 of 3 checked runners on runner-host online
· GitHub: 0 of 3 checked runners on ultra online
· mini: online
→ Wake it in person
via out of band
Expand All @@ -317,7 +317,7 @@ Spotlight is using 150% CPU — it is indexing, most likely the runner work tree
11.2% mdworker_shared
```

`--via <alias>` picks the host (default `runner-host`). `--json` prints
`--via <alias>` picks the host (default `ultra`). `--json` prints
`{ verdict, spotlightCPU, top }`. Exit `1` when Spotlight is using 50% CPU or
more, `3` when `ps` could not be run, else `0`.

Expand Down Expand Up @@ -361,12 +361,12 @@ Pairs the command line with the dashboard, with a token of its own, separate
from the app's and revocable on its own.

```bash
cockpit pair # over runner-host
cockpit pair --via runner-ts --fleet-root ~/actions-runners
cockpit pair # over ultra
cockpit pair --via ultra-ts --fleet-root ~/actions-runners
```

It runs `fleetctl.sh pair` on the host over the first `--via` alias (default
`runner-host`) to mint a six-digit code, exchanges it for a device token named
`ultra`) to mint a six-digit code, exchanges it for a device token named
"cockpit CLI on <this Mac>", and saves the token to
`~/Library/Application Support/FleetCockpit/cli-token-cli:<host>` with mode
`0600`. `--fleet-root` is where the repository lives on the host (default
Expand Down
10 changes: 5 additions & 5 deletions docs/cockpit/install.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,20 +96,20 @@ not the icon is visible.

The dashboard binds to loopback on the runner host. The default route needs
nothing changed there: the app opens an SSH tunnel through aliases from your
`~/.ssh/config`, trying each in turn — by default `runner-host`, then
`runner-ts` for a tailnet route. A minimal entry:
`~/.ssh/config`, trying each in turn — by default `ultra`, then
`ultra-ts` for a tailnet route. A minimal entry:

```
Host runner-host
HostName runner-host.local
Host ultra
HostName ultra.local
User you
IdentityFile ~/.ssh/id_ed25519
```

Test it the way the app will run it — no prompt allowed:

```bash
ssh -o BatchMode=yes runner-host 'curl -s 127.0.0.1:7878/api/health'
ssh -o BatchMode=yes ultra 'curl -s 127.0.0.1:7878/api/health'
```

If you serve the dashboard over
Expand Down
8 changes: 4 additions & 4 deletions docs/cockpit/sentinel.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ back, with how long it was gone:

| Title | Body | Priority |
|---|---|---|
| **runner-host is down** | Has not answered for 3 minutes. Asleep, off, or rebooted with nobody logged in — runners start at login. Only someone there can wake it. | urgent |
| **runner-host is back** | Answers again after 23 min. Queued jobs start on their own; dead services may need a repair. | default |
| **ultra is down** | Has not answered for 3 minutes. Asleep, off, or rebooted with nobody logged in — runners start at login. Only someone there can wake it. | urgent |
| **ultra is back** | Answers again after 23 min. Queued jobs start on their own; dead services may need a repair. | default |

Messages carry the host's name and nothing else about the fleet.

Expand All @@ -27,7 +27,7 @@ On the other machine, with a clone of this repository:
2. **Write the config** (mode `0600`):

```bash
printf '%s\n' 'SENTINEL_NAME=runner-host' 'SENTINEL_TARGET=runner' \
printf '%s\n' 'SENTINEL_NAME=ultra' 'SENTINEL_TARGET=ultra' \
"SENTINEL_NTFY=https://ntfy.sh/fleet-$(openssl rand -hex 12)" > ~/.config/fleet-sentinel.env
chmod 600 ~/.config/fleet-sentinel.env
cat ~/.config/fleet-sentinel.env # note the topic URL
Expand All @@ -51,7 +51,7 @@ another file.

| Variable | Default | Meaning |
|---|---|---|
| `SENTINEL_NAME` | `runner-host` | What the messages call the host |
| `SENTINEL_NAME` | the target | What the messages call the host |
| `SENTINEL_TARGET` | `runner` | Tailnet name or IP to probe |
| `SENTINEL_PROBE` | `tailscale` | `tailscale` (`tailscale ping` over the tailnet) or `tcp` (`nc -z <target> 22`) |
| `SENTINEL_TAILSCALE` | `/opt/homebrew/opt/tailscale/bin/tailscale` | The `tailscale` binary |
Expand Down
2 changes: 1 addition & 1 deletion docs/cockpit/settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ new build.
| Setting | Default | Meaning |
|---|---|---|
| Reach the dashboard by | SSH tunnel | **SSH tunnel**, **Direct URL** or **Fixture (demo)** |
| SSH aliases, in order | `runner-host, runner-ts` | Aliases from `~/.ssh/config`, comma- or space-separated, tried in turn |
| SSH aliases, in order | `ultra, ultra-ts` | Aliases from `~/.ssh/config`, comma- or space-separated, tried in turn |
| Dashboard port on the host | `7878` | Where the dashboard listens on the host's loopback |
| Dashboard URL | `http://127.0.0.1:7878` | For Direct URL: a dashboard served over Tailscale Serve or bound to the LAN |
| Scenario | `live` | For Fixture: which recorded state to show |
Expand Down
2 changes: 1 addition & 1 deletion docs/cockpit/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ first, then the handbook's [Troubleshooting](../troubleshooting.md).
Run the connection by hand, exactly as the app does:

```bash
ssh -o BatchMode=yes runner-host 'curl -s -o /dev/null -w "%{http_code}\n" 127.0.0.1:7878/api/health'
ssh -o BatchMode=yes ultra 'curl -s -o /dev/null -w "%{http_code}\n" 127.0.0.1:7878/api/health'
```

| You see | Cause | Fix |
Expand Down
9 changes: 9 additions & 0 deletions docs/federation.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ Federation lets the dashboard coordinate runners across several Macs. Each
extra Mac runs a lightweight agent (`agent.js`) that reports outbound to the
coordinator — no inbound port is needed on agent hosts.

!!! note "Current topology (2026-10-21)"
One Mac, `ultra`, is both the coordinator (fleetd, replica id `ultra`) and
the only runner host, with `FLEET_TIERS_MODE=off`. The previous runner host
was retired after [moving the coordinator](#moving-the-coordinator-to-another-mac).
Fleet Cockpit reaches the dashboard through the ssh aliases `ultra`, then
`ultra-ts`; a different or extra coordinator is a setting there, not a new
build. To add a runner host, follow
[Adding a second runner host](#adding-a-second-runner-host).

## Architecture

![Two agent Macs above a coordinator Mac and a browser. Each agent runs agent.js and its own runners and opens no inbound port; its only connection is an outbound heartbeat every 30 seconds, with commands riding back in the reply. The second agent has been silent for six minutes and is marked stale, its runners greyed rather than removed. The coordinator runs fleetd, the dashboard and its own runners. Side notes say each host computes its own headroom, and that a coordinator may drain, resume, switch off and health-check an agent's runners, but removing one needs that host's own opt-in](img/federation.png)
Expand Down
4 changes: 2 additions & 2 deletions docs/reference/scripts.md
Original file line number Diff line number Diff line change
Expand Up @@ -1205,9 +1205,9 @@ Source: [`dashboard/scripts/settle-stale-jobs.mjs`](https://github.com/addisdev/

```bash
cd dashboard
ssh runner-host "/usr/bin/sqlite3 -json ~/actions-runners/dashboard/fleet.db \"$(node scripts/settle-stale-jobs.mjs query)\"" > stale.json
ssh ultra "/usr/bin/sqlite3 -json ~/actions-runners/dashboard/fleet.db \"$(node scripts/settle-stale-jobs.mjs query)\"" > stale.json
node scripts/settle-stale-jobs.mjs fetch --in stale.json --out fetched.ndjson
node scripts/settle-stale-jobs.mjs sql --in fetched.ndjson | ssh runner-host /usr/bin/sqlite3 ~/actions-runners/dashboard/fleet.db
node scripts/settle-stale-jobs.mjs sql --in fetched.ndjson | ssh ultra /usr/bin/sqlite3 ~/actions-runners/dashboard/fleet.db
```

### `dashboard/scripts/mirror-plan.mjs`
Expand Down
2 changes: 1 addition & 1 deletion docs/tools/shoot-motion.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ const opt = (name, fallback = null) => {
return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : fallback;
};

const SSH_HOST = opt('host', process.env.MOTION_SSH_HOST ?? 'runner-host');
const SSH_HOST = opt('host', process.env.MOTION_SSH_HOST ?? 'ultra');
const RUNNER_DIR = opt('runner', process.env.MOTION_RUNNER ?? null);
const FLEET_PORT = Number(opt('fleet-port', process.env.MOTION_FLEET_PORT ?? 7878));
const SERVE_ONLY = flag('serve');
Expand Down
Loading
Loading