From ec81e9433dc03ddadff4d8d65c3cc8e90d05f880 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Tue, 13 Oct 2026 17:36:26 -0400 Subject: [PATCH 1/8] Add discovery record identity contracts Suggested review order: - services/nvpair-node-scanner/spec.md, services/nvpair-node-info/spec.md, and services/nvpair-manual-nodes/spec.md - the complete final behavior contract, implemented by the following commits. - services/shared/noderec/noderec_test.go - services/shared/noderec/noderec.go - services/nvpair-node-scanner/directory_test.go - services/nvpair-node-scanner/directory.go - discovery record IDs track candidates, without authenticating their identity. - services/nvpair-node-scanner/daemon_test.go - services/nvpair-node-scanner/daemon.go - services/nvpair-ui-broker/discovery_test.go - services/nvpair-ui-broker/discovery.go - services/nvpair-ui-broker/manualnodes.go - services/nvpair-ui-broker/broker.go - desktop/tests/modular/node-identity-contract.test.ts - desktop/src/shared/types/cluster.ts - desktop/src/electron/service-bridge/modular-state.ts - services/nvpair-tui/ui/nodeswire_test.go - services/nvpair-tui/ui/nodeswire.go - desktop/docs/frontend-api.md - services/nvpair-node-scanner/README.md - services/nvpair-ui-broker/README.md Signed-off-by: Kaylee Lubick --- desktop/docs/frontend-api.md | 4 + .../electron/service-bridge/modular-state.ts | 30 +- desktop/src/shared/types/cluster.ts | 6 + .../modular/node-identity-contract.test.ts | 93 ++ services/nvpair-manual-nodes/spec.md | 104 +-- services/nvpair-node-info/spec.md | 125 +-- services/nvpair-node-scanner/README.md | 7 + services/nvpair-node-scanner/daemon.go | 18 +- services/nvpair-node-scanner/daemon_test.go | 9 + services/nvpair-node-scanner/directory.go | 30 +- .../nvpair-node-scanner/directory_test.go | 67 ++ services/nvpair-node-scanner/spec.md | 874 ++++++++++-------- services/nvpair-tui/ui/nodeswire.go | 10 +- services/nvpair-tui/ui/nodeswire_test.go | 22 + services/nvpair-ui-broker/README.md | 6 + services/nvpair-ui-broker/broker.go | 6 +- services/nvpair-ui-broker/discovery.go | 60 +- services/nvpair-ui-broker/discovery_test.go | 20 + services/nvpair-ui-broker/manualnodes.go | 20 +- services/shared/noderec/noderec.go | 41 +- services/shared/noderec/noderec_test.go | 31 + 21 files changed, 1045 insertions(+), 538 deletions(-) create mode 100644 desktop/tests/modular/node-identity-contract.test.ts create mode 100644 services/nvpair-tui/ui/nodeswire_test.go diff --git a/desktop/docs/frontend-api.md b/desktop/docs/frontend-api.md index 6209ca269..c33579d52 100644 --- a/desktop/docs/frontend-api.md +++ b/desktop/docs/frontend-api.md @@ -71,6 +71,10 @@ They do not imply a WebSocket connection. Browser clients are not supported. `name` (hostname), the reachable `ipAddress`/`port`, and the `trusted` and `clustered` flags. A `clustered` node already belongs to some cluster and is rendered non-invitable. + Broker records also carry `discoveryRecordId` and `identityVerification` (`local`, + `pinned`, or `unverified`). The bridge retains these provenance fields through + proxy updates and clears them when the broker withdraws the record; public + hardware readings cannot change them. - `onNodesChanged(callback)` reports discovery changes. ### `pairApi.engines` diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index 0c7390bf7..f512de566 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -15,7 +15,7 @@ import type { NodeItemMetrics } from '@/shared/types/metrics' import type { NodeItem } from '@/shared/types/nodes' import type { ServiceError, ServiceErrorAction, ServiceErrorSeverity } from '@/shared/types/errors' import type { Workload, WorkloadState } from '@/shared/types/workloads' -import type { ClusterNode, Invite } from '@/shared/types/cluster' +import type { ClusterNode, Invite, NodeIdentityVerification } from '@/shared/types/cluster' import type { WsInvokeResponse } from '@/shared/types/ws-channels' import { MODULAR_NODE_INFO_SELF_HOST, @@ -82,6 +82,8 @@ interface RemoteEngineFacts { } interface ModularNode { + discoveryRecordId?: string + identityVerification?: NodeIdentityVerification // Canonical cross-domain node key = the backend's stable per-host UUID // (`shared/nodeid`). The broker carries it on `AvailableNode.hostUuid`, the // proxies key `Node.ID` by it, and `/v1/node-info` reports it, so every @@ -752,6 +754,12 @@ function parseProxyNode(params: JsonValue | undefined, engine: ProxyEngine): Mod } } +function identityVerificationValue( + value: JsonValue | undefined +): NodeIdentityVerification | undefined { + return value === 'local' || value === 'pinned' || value === 'unverified' ? value : undefined +} + function parseBrokerNode(params: JsonValue | undefined): ModularNode | null { const obj = objectValue(params) if (!obj) return null @@ -768,8 +776,12 @@ function parseBrokerNode(params: JsonValue | undefined): ModularNode | null { // when it published a single address (where it would only repeat ipAddress). const ipAddresses = stringArrayValue(obj.ipAddresses) const brokerAddresses = ipAddresses.length > 0 ? ipAddresses : ipAddress ? [ipAddress] : [] + const discoveryRecordId = stringValue(obj.discoveryRecordId) + const identityVerification = identityVerificationValue(obj.identityVerification) return { id: hostUuid, + ...(discoveryRecordId ? { discoveryRecordId } : {}), + ...(identityVerification ? { identityVerification } : {}), sources: ['broker'], name: hostname, host: ipAddress, @@ -837,6 +849,8 @@ function sameEngines( function sameNode(left: ModularNode, right: ModularNode): boolean { return ( left.id === right.id && + left.discoveryRecordId === right.discoveryRecordId && + left.identityVerification === right.identityVerification && sameStringList(left.sources, right.sources) && left.name === right.name && left.host === right.host && @@ -1132,6 +1146,10 @@ class ModularBridgeState { const addresses = nodeAddresses(node) return { id: node.id, + ...(node.discoveryRecordId ? { discoveryRecordId: node.discoveryRecordId } : {}), + ...(node.identityVerification + ? { identityVerification: node.identityVerification } + : {}), // Display label: hostname, else reachable address; never the raw UUID. name: node.name || primaryNodeAddress(node) || node.id, ipAddress: primaryNodeAddress(node), @@ -2374,6 +2392,10 @@ class ModularBridgeState { memory: null, lastSeen: Date.now() } + // The spread copied broker-owned provenance. Remove it when the broker + // withdraws the record; surviving proxy presence cannot preserve verification. + delete next.discoveryRecordId + delete next.identityVerification this.nodes.set(next.id, next) this.emitNodeChanged(next) } @@ -2566,6 +2588,12 @@ class ModularBridgeState { // broker-reported values rather than overwriting them with the proxy // node's empty name and default-false flags. name: existing.name || next.name, + ...(existing.discoveryRecordId + ? { discoveryRecordId: existing.discoveryRecordId } + : {}), + ...(existing.identityVerification + ? { identityVerification: existing.identityVerification } + : {}), trusted: existing.trusted, clustered: existing.clustered, // Symmetrically: this event is the proxy feed's whole list (carried by diff --git a/desktop/src/shared/types/cluster.ts b/desktop/src/shared/types/cluster.ts index e02daa241..6d5f7d9d5 100644 --- a/desktop/src/shared/types/cluster.ts +++ b/desktop/src/shared/types/cluster.ts @@ -10,7 +10,13 @@ * interactive PIN-pairing invite handshake. */ +export type NodeIdentityVerification = 'local' | 'pinned' | 'unverified' + export interface AvailableNode { + /** Discovery record identity, independent of a claimed host UUID. */ + discoveryRecordId?: string + /** How the service established this record's identity. */ + identityVerification?: NodeIdentityVerification /** Canonical node key = the backend's stable per-host UUID (matches `NodeItem.id`). */ id: string /** Display hostname. */ diff --git a/desktop/tests/modular/node-identity-contract.test.ts b/desktop/tests/modular/node-identity-contract.test.ts new file mode 100644 index 000000000..5d1fd7560 --- /dev/null +++ b/desktop/tests/modular/node-identity-contract.test.ts @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) +vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) + +import { getModularBridgeState } from '@/electron/service-bridge/modular-state' + +describe('node identity contract', () => { + it('clears withdrawn broker provenance while proxy presence survives', () => { + const state = getModularBridgeState() + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [ + { + hostUuid: 'identity-withdrawn-host', + name: 'workstation', + ipAddress: '192.0.2.21', + port: 14318, + discoveryRecordId: 'peer:identity-withdrawn-host', + identityVerification: 'pinned' + } + ] + }) + state.handleNotification({ + source: 'ollama-proxy', + method: 'node/discovered', + params: { + id: 'identity-withdrawn-host', + host: 'workstation', + port: 11434, + addresses: ['192.0.2.21'] + } + }) + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [] + }) + + const node = state.getAvailableNodes().find(node => node.id === 'identity-withdrawn-host') + expect(node).toBeDefined() + expect(node).not.toHaveProperty('discoveryRecordId') + expect(node).not.toHaveProperty('identityVerification') + }) + + it('preserves broker provenance through proxy and public hardware updates', () => { + const state = getModularBridgeState() + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [ + { + hostUuid: 'identity-contract-host', + name: 'workstation', + ipAddress: '192.0.2.20', + port: 14318, + discoveryRecordId: 'observation:candidate-a', + identityVerification: 'unverified' + } + ] + }) + state.handleNotification({ + source: 'ollama-proxy', + method: 'node/discovered', + params: { + id: 'identity-contract-host', + host: 'workstation', + port: 11434, + addresses: ['192.0.2.20'] + } + }) + state.mergeNodeInfoResponse('identity-contract-host', { + hostUuid: 'identity-contract-host', + discoveryRecordId: 'peer:identity-contract-host', + identityVerification: 'pinned', + GPUs: [{ name: 'Display GPU' }] + }) + + expect( + state.getAvailableNodes().find(node => node.id === 'identity-contract-host') + ).toMatchObject({ + discoveryRecordId: 'observation:candidate-a', + identityVerification: 'unverified' + }) + expect(state.getNodesInitial().nodes['identity-contract-host'].topology.gpus).toHaveLength( + 1 + ) + }) +}) diff --git a/services/nvpair-manual-nodes/spec.md b/services/nvpair-manual-nodes/spec.md index 5d93453ce..0d1508783 100644 --- a/services/nvpair-manual-nodes/spec.md +++ b/services/nvpair-manual-nodes/spec.md @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 `nvpair-manual-nodes` owns the set of user-entered peer node addresses on networks where mDNS discovery is unavailable, filtered, or incomplete. A client hands it a bare host (`node/add`), and it becomes the authority for whether that host is currently reachable and what it serves: it probes each entered address for an Ollama engine, an LM Studio engine, and an `nvpair-node-info` telemetry endpoint every ten seconds, and reports the aggregate as a `ManualNodeStatus` on `node/discovered` / `node/updated` / `node/removed`. Its purpose is to make an address the user asserts routable — the supervising `nvpair-ui-broker` merges its events into the same discovery snapshot `nvpair-node-scanner` feeds, bridges reachable manual nodes into the matching Ollama or LM Studio facade of `nvpair-proxy`, and feeds their GPU telemetry to `nvpair-job-scheduler`, so for those two engines a manual node is routable and schedulable like a discovered one. It does not cover llama.cpp: a manual-only node is never a llama.cpp target (§4). -It exists because `nvpair-node-scanner` has no manual concept at all: the scanner only reports what it browses over mDNS, and it has no `node/add` surface. Every non-multicast path into the fleet is this worker's. It is deliberately a probe engine and nothing more — it opens no network listener, keeps its entries only in `map[string]*trackedNode`, and writes no file. The durable list and its replay after a restart live in the PAIR desktop app (§7.6, §9). +It exists because `nvpair-node-scanner` has no manual concept at all: the scanner only reports what it browses over mDNS, and it has no `node/add` surface. Every non-multicast path into the fleet is this worker's. It is deliberately a probe engine and nothing more — it opens no network listener, keeps its entries only in `map[string]*trackedNode`, and writes no file. The supervising broker owns the durable list and replays it after a restart (§7.6, §9). ## 2. Scope @@ -18,12 +18,12 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne - Accept, track, and drop user-entered node addresses (`node/add`, `node/remove`, `nodes/list`). - Probe each tracked address on a fixed ten-second cycle for Ollama, LM Studio, and `nvpair-node-info`. - Derive per-engine model inventories and node-info telemetry (GPU, CPU, memory, GPU sample validity and age, `hostUuid`) from those probes. -- Emit change-gated status notifications and, on sustained unreachability, `errors:report` / `errors:clear`. -- Choose the probe transport for the node-info leg: plain HTTP, an operator-supplied TLS client, or cluster mTLS. +- Emit changed engine status and every accepted hardware snapshot, plus sustained-unreachability `errors:report` / `errors:clear`. +- Require exact current cluster pins for every service node-info read; keep public HTTP hardware displays in desktop/TUI. **Out of scope** -- Durable storage of the entry list — the desktop app persists `manual-nodes.json` and replays it (§7.6); neither this worker nor `nvpair-ui-broker` holds an authoritative copy. +- Durable storage of the entry list — `nvpair-ui-broker` persists and replays `manual-nodes.json` (§7.6); this worker holds only current probe state. - mDNS browsing and advertisement — `nvpair-node-scanner`. - Merging manual and discovered nodes into one snapshot, the per-engine proxy bridge, and the scheduler telemetry feed — `nvpair-ui-broker`. - Inference routing and node ranking — `nvpair-proxy` and `nvpair-job-scheduler`. @@ -34,28 +34,27 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne ## 3. Key Use Cases - **Add a node**: a client sends `node/add` with `{"address": "10.0.1.50"}`. The worker upserts the entry, responds immediately with an unprobed `ManualNodeStatus` (ports filled, all `*_up` false), then probes on a goroutine and emits `node/discovered` with the real outcome. -- **Track reachability**: every `probeInterval` (10 s) the probe loop walks a snapshot of the entries and re-probes each. A `node/updated` fires only when a watched field actually changed (§7.2). -- **Learn a stable identity**: once the remote's `/v1/node-info` answers with `hostUuid`, that value rides on every subsequent status, and `nvpair-ui-broker` rekeys the node from the manual id onto the real host UUID — collapsing a machine that is both manually added and mDNS-discovered into one entry. +- **Track reachability**: every `probeInterval` (10 s), re-probe the entries. Publish changed engine status and every accepted hardware snapshot, including unchanged measurements. +- **Learn a stable identity**: pinned mTLS and matching `hostUuid` / `clusterUuid` bind the response to its certificate. The broker may then merge this manual source with the same verified host. Unverified configurations retain independent keys. - **Remove a node**: `node/remove` with `{"id": "lab"}` deletes the tracked entry, emits `node/removed` with just the id, and fires a defensive `errors:clear` for that node's probe-failed id. - **List for a UI**: `nvpair-tui`'s Nodes tab re-lists `nodes/list` every 10 s and merges the hand-added entries into its one node table, so their reachability stays current. - **Edge case — probe still in flight when removed**: `node/remove` between `node/add` and its initial probe completing wins. The probe goroutine re-reads the map, finds the id gone, and returns without emitting `node/discovered`; `probeNode` likewise drops its result if the id vanished mid-probe. -- **Edge case — node-info blip while an engine stays up**: a failed node-info probe returns no `hostUuid`. The previously learned value is carried forward rather than blanked, so the broker's store key does not flap from UUID to manual id and back. +- **Edge case — node-info blip while an engine stays up**: clear the failed measurement and its verification. The explicitly configured engine remains independently routable when it answers; no public hardware fallback supplies scheduling data. - **Edge case — engine reachable, model list unparseable**: LM Studio answering `200` with a body that fails to decode is still reported `lmstudio_up: true` with no models. Ollama answering `200` on `/` but failing `/api/tags` is `ollama_up: true` with no models. -- **Edge case — duplicate add**: `node/add` with a name (or address) that resolves to an existing id overwrites that entry's `ManualEntry` in place and re-probes it. The same address added under two different names produces two independent ids that only collapse downstream, after `hostUuid` is learned. +- **Edge case — duplicate add**: `node/add` with a name (or address) that resolves to an existing id overwrites that entry's `ManualEntry` in place and re-probes it. The same address added under two different names produces two independent ids that only collapse downstream, after `hostUuid` is authenticated. - **Edge case — malformed params**: `node/add` whose params are not an object, or whose `address` is empty, is rejected `-32602` and adds nothing. - **Edge case — notification instead of request**: an id-less `node/add` frame is logged and ignored; state is not mutated and nothing is emitted. ## 4. Open Questions / Risks -- **Entry durability (known limitation — fix in progress)**: the worker keeps entries only in memory and `nvpair-ui-broker` keeps no authoritative copy, so a manual-nodes restart loses the user's nodes. The broker's `clearManualNodesState` evicts the orphaned entries from the discovery store and the Ollama and LM Studio facades so the view stays honest, and the client must re-add them. This is a known limitation, and open pull requests will fix it soon. -- **Durable list has readers but no writer (open — needs a product decision)**: the desktop app reads `manual-nodes.json` (`listManualNodeEntries`) and replays it through `node/add` after broker ready, and prunes it on member removal (`removeManualNodeEntry`), but no code path appends a newly added node to it. Entries added through `nvpair-tui`'s `node/add` are never persisted. In practice the replay list stays empty unless the file is authored by hand. +- **Worker restart**: current probe results are lost. The broker withdraws the retired worker's records and samples, then replays durable entries to the replacement worker. Restored configurations start unverified and must authenticate again before supplying hardware data. - **Risk — `node/updated` on every probe**: change detection compares `cpu`, `memory`, and `gpus` by value, and also `telemetryValid` and `msSince`. Because `msSince` is the remote GPU sample's age at response time, it differs on essentially every probe, so any node whose `nvpair-node-info` reports valid GPU telemetry emits a full `node/updated` every 10 s. Each one makes the broker re-project the discovery snapshot, re-bridge the node into `nvpair-proxy`, and re-ingest its telemetry. - **llama.cpp is not supported (open — gap)**: llama.cpp is the third registered engine, but the prober checks only `11434` and `1234`, `ManualNodeStatus` has no llama.cpp fields, and the broker's `bridgeManualNode`, `removeManualNodeFromProxies`, and `manualModelsByEngine` handle only `ollama` and `lmstudio`. A manual-only node therefore never enters the llama.cpp facade's candidate set and advertises no llama.cpp models. A machine that mDNS also discovers still reaches that facade through the broker's discovery relay under its `lc` record. -- **Risk — fixed engine and telemetry ports**: `11434` (Ollama), `1234` (LM Studio), and `14318` (plain-HTTP node-info) are compiled in. A remote engine on a non-default port is invisible, and an address entered as `host:port` produces a malformed probe URL, so the node reads permanently down. Only `tls_port` lets a caller move a probe port, and only for the node-info leg. +- **Risk — fixed engine and telemetry ports**: `11434` (Ollama), `1234` (LM Studio), and `14318` (pinned node-info) are compiled in. A remote engine on a non-default port is invisible, and an address entered as `host:port` produces a malformed probe URL, so the node reads permanently down. Only `tls_port` lets a caller move a probe port, and only for the node-info leg. - **Risk — non-EOF read errors loop**: `readLoop` treats every non-`io.EOF` read error as recoverable and continues. `nvpair-shared/jsonrpc` distinguishes a per-frame `DecodeError` from a terminal transport error, but the manager ignores that distinction, so a permanently failing pipe spins the loop logging on each iteration. -- **Risk — IP-literal entries cannot self-heal**: an entry added by raw IP is dead once the device is reassigned an address; only a hostname entry re-resolves (keep-alives are disabled precisely so each probe re-resolves). The probe-failed message says so, but nothing re-discovers the node. +- **Risk — IP-literal entries cannot self-heal**: an entry added by raw IP is dead once the device is reassigned an address; only a hostname entry re-resolves (engine keep-alives are disabled so each engine probe re-resolves). The probe-failed message says so, but nothing re-discovers the node. - **Risk — no address validation**: any non-empty string is accepted as an address. A typo is indistinguishable from a powered-off node until the 30-second failure threshold reports it. -- **Risk — inert `mtls` field**: `mtls` is persisted and echoed as `mtls_required`, but the probe path never reads it; the transport decision is driven entirely by `tls_port` and the cluster mesh. A caller could reasonably read it as a requirement it is not. +- **Configuration hints**: `tls_port` selects the node-info port and `mtls` is echoed as a configuration hint. Hardware reads always require pinned mTLS, including when both hints are absent. Neither hint changes the independently configured plain HTTP engine routes. - **Firewall inventory divergence (open — needs reconciliation)**: the Windows NSIS installer adds an inbound rule `"NVPAIR Manual Nodes"` for a binary that opens no listening socket, while `MODULAR_RUNTIME_BINARIES` marks it `needsFirewallAccess: false` and the macOS privileged helper grants it no rule. The inbound rule is unnecessary. - **Future — engine port discovery**: if `nvpair-engine-manager` ever reports remote engine ports, the hardcoded `11434` / `1234` probes should be replaced by the reported values rather than extended with more entry-level hints. @@ -66,9 +65,9 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne - Accept `node/add` with a required `address` and optional `name`, `tls_port`, `mtls`; reject an absent or empty `address`. - Key each entry by `nodeID(entry)`: the caller's `name` when non-empty, else `"manual:" + address`. - Respond to `node/add` before probing, with the entry's ports and TLS flags already populated. -- Probe every tracked entry every `probeInterval` (10 s) with a `probeTimeout` (3 s) per HTTP request and idle connection reuse disabled. -- Emit `node/discovered` once after an entry's initial probe, and `node/updated` on every subsequent probe whose watched fields changed. -- Preserve the last-learned `hostUuid` when a node-info probe fails and reports none. +- Probe every tracked entry every `probeInterval` (10 s), with a `probeTimeout` (3 s) per request. Engine probes disable idle reuse; known peer hardware reads use shared pooled clients. +- Emit `node/discovered` after the initial probe, and `node/updated` when watched state changes or a new authenticated hardware sample arrives. +- Clear measurements and verification on failed authenticated reads. Local measurements belong exclusively to owned-worker IPC, including for manually entered loopback endpoints. - Emit exactly one `errors:report` per failure episode, on the probe where `consecutiveFails` first reaches `probeFailThreshold` (3), and one `errors:clear` on the first reachable probe after having been at or above the threshold. - Emit `errors:clear` for a removed node's probe-failed id unconditionally on `node/remove`. - Return `{"removed": false}` for `node/remove` of an unknown id, without emitting `node/removed`. @@ -96,7 +95,7 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne { "address": "10.0.1.50", "name": "lab", - "tls_port": 14319, + "tls_port": 14318, "mtls": true } ``` @@ -130,7 +129,9 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne "memory": { "total_bytes": 137438953472, "used_bytes": 34359738368 }, "telemetryValid": true, "msSince": 137, - "hostUuid": "b1f0c2de-8a44-4d0e-9c1a-77e2f5b3a901" + "hostUuid": "b1f0c2de-8a44-4d0e-9c1a-77e2f5b3a901", + "discoveryRecordId": "peer:b1f0c2de-8a44-4d0e-9c1a-77e2f5b3a901:opaque-binding", + "identityVerification": "pinned" } ``` @@ -138,10 +139,12 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne `telemetryValid` and `msSince` are always present. They copy node-info's GPU sample validity and the sample's age in milliseconds, preserving its distinction between an idle 0% reading and no reading at all, and they read `false` and `0` whenever node-info is down. Consumers ignore `msSince` while `telemetryValid` is false. +`discoveryRecordId` is an opaque binding to the current local identity and admission and the exact peer pin. It is present only after authentication. Removed or replaced authorization clears that binding, `hostUuid`, and service hardware measurements. Cached status reads and notifications include elapsed time since the accepted hardware response. + Example emitted frame (`node/discovered`, an unnamed entry whose LM Studio is absent): ```json -{"jsonrpc":"2.0","method":"node/discovered","params":{"id":"manual:10.0.1.50","address":"10.0.1.50","ollama_up":true,"ollama_port":11434,"ollama_models":["llama3.2:latest"],"lmstudio_up":false,"lmstudio_port":1234,"node_info_up":false,"node_info_port":14318,"telemetryValid":false,"msSince":0}} +{"jsonrpc":"2.0","method":"node/discovered","params":{"identityVerification":"unverified","id":"manual:10.0.1.50","address":"10.0.1.50","ollama_up":true,"ollama_port":11434,"ollama_models":["llama3.2:latest"],"lmstudio_up":false,"lmstudio_port":1234,"node_info_up":false,"node_info_port":14318,"telemetryValid":false,"msSince":0}} ``` ## 7. API / Interface Contract @@ -158,31 +161,29 @@ One interface: newline-delimited JSON-RPC 2.0 over `stdin`/`stdout`, or over the | `shutdown` | none | `null` | | `log/set-level` | `{ "level": "debug"\|"info"\|"warn"\|"error" }` | `{ "level": string }` | -Notifications emitted: `ready` with `{ "version": string }` (once, before the probe loop starts); `node/discovered` with a full `ManualNodeStatus` (once per entry, after its initial probe); `node/updated` with a full `ManualNodeStatus` (on a changed re-probe); `node/removed` with a `ManualNodeStatus` carrying only `id`, plus `address`, `ollama_up`, `ollama_port`, `lmstudio_up`, `lmstudio_port`, `node_info_up`, `node_info_port`, `telemetryValid`, and `msSince` at their zero values; `errors:report` with an `nvpair-shared/errors` `ServiceError`; and `errors:clear` with a `ClearParams` (`{ "id": string }`). +Notifications emitted: `ready` with `{ "version": string }` (once, before the probe loop starts); `node/discovered` with a full `ManualNodeStatus` (once per entry, after its initial probe); `node/updated` with a full `ManualNodeStatus` (on changed state or every accepted hardware sample); `node/removed` with a `ManualNodeStatus` carrying only `id`, plus `address`, `ollama_up`, `ollama_port`, `lmstudio_up`, `lmstudio_port`, `node_info_up`, `node_info_port`, `telemetryValid`, and `msSince` at their zero values; `errors:report` with an `nvpair-shared/errors` `ServiceError`; and `errors:clear` with a `ClearParams` (`{ "id": string }`). ### 7.1 Address entry, identity, and idempotency -`nodeID(entry)` is the entry's identity: `name` when non-empty, otherwise `"manual:" + address`. Adds are upserts into `map[string]*trackedNode` — re-adding the same id replaces its `ManualEntry` and its probe baseline, so a second `node/add` under an existing name silently repoints that id at the new address. Two names for one address are two independent entries; each probes the host separately, and `nvpair-ui-broker` collapses them onto one discovery key only once `hostUuid` is learned. Removal is idempotent: the second `node/remove` of an id returns `{"removed": false}` and emits nothing. +`nodeID(entry)` is the entry's identity: `name` when non-empty, otherwise `"manual:" + address`. Adds are upserts into `map[string]*trackedNode` — re-adding the same id replaces its `ManualEntry` and its probe baseline, so a second `node/add` under an existing name silently repoints that id at the new address. Two names for one address are two independent entries; each probes the host separately, and the broker joins them to one canonical host only after authentication. Removal is idempotent: the second `node/remove` of an id returns `{"removed": false}` and emits nothing. Broker clients select exactly one of worker entry `id` or broker node key `nodeId`; the worker itself accepts `id` only. ### 7.2 Probe protocol and reachability -Each probe cycle performs up to four HTTP GETs against the entered address, each bounded by `probeTimeout` (3 s) on a transport with `DisableKeepAlives: true` so every cycle re-resolves the address: +Each probe cycle performs up to four HTTP GETs against the entered address, each bounded by `probeTimeout` (3 s). Engine probes disable keep-alives so each cycle re-resolves the address. Hardware reads use the transport in §7.3: - Ollama: `GET http://
:11434/` must return `200`; on success `GET http://
:11434/api/tags` supplies `ollama_models` from `models[].name`. - LM Studio: `GET http://
:1234/v1/models` must return `200`; the same response supplies `lmstudio_models` from `data[].id`. -- node-info: `GET ://
:/v1/node-info` must return `200` and decode into `NodeInfoResponse` (`GPUs`, `cpu`, `memory`, `telemetryValid`, `msSince`, `hostUuid`). A decode failure counts as down. +- node-info: `GET https://
:/v1/node-info` must return `200`, fit within 64 KiB, and decode into `NodeInfoResponse`. Require an exact current pin, matching host and cluster response identities, and a remote identity distinct from this host. -`node_info_port` is `14318` over `http` unless the entry set `tls_port > 0`, in which case it is `tls_port` over `https`. The two engine legs are always plain HTTP. `reachable` is `ollama_up || lmstudio_up || node_info_up`. A `node/updated` fires only when `ollama_up`, `lmstudio_up`, `node_info_up`, `hostUuid`, `ollama_models`, `lmstudio_models`, `gpus`, `cpu`, `memory`, `telemetryValid`, or `msSince` differs from the previous status; a stable probe logs at debug and emits nothing. In practice `msSince` makes every probe of a node with valid GPU telemetry a change (§4). +`node_info_port` is `14318` unless `tls_port > 0` overrides it. Hardware reads always use pinned mTLS. Engine legs remain explicitly configured plain HTTP. `reachable` is `ollama_up || lmstudio_up || node_info_up`. Publish changed status and every accepted hardware snapshot so stable values do not stop refreshing the scheduling cache. -### 7.3 Probe transport: operator TLS and cluster mTLS +### 7.3 Probe transport and authenticated identity -Three transports exist for the node-info leg, selected in this order: +Refresh the live cluster mesh before reading hardware. Without membership and an approved certificate, no service hardware request is made. For an unknown manual endpoint, `ClientTLSConfigAny()` presents this node's leaf and accepts a currently pinned certificate. Subsequent reads reuse the shared peer-client pool for that proved principal. Recheck the actual response certificate against current pins after reading the body, and require both response identities to match it. Self certificates cannot supply local service measurements over HTTP. -1. **Plain HTTP** (`tls_port` absent or zero) — the default client, port `14318`. -2. **Operator-supplied TLS** (`tls_port > 0`) — the client built from `--client-cert` / `--client-key` / `--ca-bundle`, at TLS 1.2 minimum. `--ca-bundle` is appended to the system pool, so enterprise and public roots both work; `--client-cert` requires `--client-key` and vice versa, and a mismatch is a fatal startup error. With none of the three set, the client falls back to Go's default system trust store, so an HTTPS node with a publicly trusted certificate is probed with no flags at all. -3. **Cluster mTLS** (`tls_port > 0` and `--cluster-dir` resolves to a clustered mesh) — `clustertrust.Mesh.Refresh()` runs first so a cluster joined or a peer paired after startup is picked up without a restart, then `ClientTLSConfigAny()` yields a client presenting this node's leaf and accepting any currently-pinned peer certificate by exact DER match. This is the only way to reach a clustered peer's node-info, which is pin-gated and serves no plaintext listener. A manual node carries no discovery record to key a specific pin on, hence "any pinned cert". +Capture the authorization binding before writing the request, including on reused connections, and recheck it before publishing the result. A local identity or admission change, peer pin replacement or removal, or replacement of the tracked manual entry invalidates in-flight hardware results. A separate trust watcher withdraws cached verification independently of slow engine probes. A retired worker's callback cannot restore state in the broker. -A successful probe proves only that *something* at that address answered an expected HTTP shape. On the plain-HTTP and operator-TLS paths nothing authenticates the peer to PAIR, and `hostUuid` is self-reported by the remote. Only the cluster-mTLS path binds the answer to a certificate `nvpair-cluster-manager` already pinned. A manually entered address is therefore a user assertion of reachability, never proof of a trusted peer. +Redirects are refused. Request deadlines and response-size limits apply, and sample age includes request time. Failed reads never downgrade to public HTTP or operator-CA TLS. Public HTTP remains available to desktop and TUI for display only. Explicit manual engine configuration independently authorizes the engine probes and routes, never the use of public hardware data. ### 7.4 Failure reporting through the errors pipeline @@ -190,15 +191,15 @@ A successful probe proves only that *something* at that address answered an expe ### 7.5 Downstream projection -`nvpair-ui-broker` consumes the notifications in `forwardManualNodesNotification`. `node/discovered` and `node/updated` go through `manualToEnriched` into the same discovery store `nvpair-node-scanner` feeds, under source `sourceManual`: `Host` and `Addresses` come from `address`, `Port` from `node_info_port`, `Models` from the de-duplicated union of both engines' lists, and `ModelsByEngine` from `{"ollama": …, "lmstudio": …}`. The store key is `hostUuid` when reported, else the manual id, so manual and mDNS nodes share one `discovery:get-nodes` / `discovery:nodes-changed` snapshot. In parallel, `bridgeManualNode` upserts the node into the `nvpair-proxy` Ollama and LM Studio facades whose engine is reachable, with the engine-addressed `ollama:node/add-manual` or `lmstudio:node/add-manual`, and withdraws it from either facade whose engine is not with `:node/remove-manual`. The llama.cpp facade receives no manual nodes (§4). The candidate is keyed identically, so `nvpair-job-scheduler` priority and `scheduledOn` resolve to it. +`nvpair-ui-broker` consumes the notifications in `forwardManualNodesNotification`. `node/discovered` and `node/updated` go through `manualToEnriched` into the same discovery store `nvpair-node-scanner` feeds, under source `sourceManual`: `Host` and `Addresses` come from `address`, `Port` from `node_info_port`, `Models` from the de-duplicated union of both engines' lists, and `ModelsByEngine` from `{"ollama": …, "lmstudio": …}`. The store key is the authenticated `hostUuid`, otherwise the independent `manual:` record key, so manual and mDNS nodes share one `discovery:get-nodes` / `discovery:nodes-changed` snapshot. In parallel, `bridgeManualNode` upserts the node into the `nvpair-proxy` Ollama and LM Studio facades whose engine is reachable, with the engine-addressed `ollama:node/add-manual` or `lmstudio:node/add-manual`, and withdraws it from either facade whose engine is not with `:node/remove-manual`. The llama.cpp facade receives no manual nodes (§4). The candidate is keyed identically, so `nvpair-job-scheduler` priority and `scheduledOn` resolve to it. -The broker also reduces each status to the highest `utilization_percent` across its GPUs plus `telemetryValid` and `msSince`, and caches that as the manual telemetry source for the same key. A scanner observation of the same host takes precedence. The broker ages the cached sample and relays it to `nvpair-job-scheduler`, which is how a manual node gets GPU pressure in scheduling. +The broker reduces only pinned statuses to the highest `utilization_percent` across its GPUs plus `telemetryValid` and `msSince`, and caches that as the manual telemetry source for the same key. A scanner observation of the same host takes precedence. The broker ages the cached sample and relays it to `nvpair-job-scheduler`, which is how a manual node gets GPU pressure in scheduling. `node/removed` drops the alias and either reprojects the key from a surviving alias or releases the store, telemetry, and proxy claims. `nvpair-node-scanner` participates in none of this — it has no manual concept. ### 7.6 Durable entry list and replay -This worker persists nothing. The durable list is a JSON array of `{ id, address, name }` at `configs/manual-nodes.json` under the PAIR desktop user-data directory (§9), written atomically (temp file plus rename). The desktop app reads it after broker `app:ready` and replays each entry as `node/add` with `{ address, name }`, and prunes an entry when the corresponding node is removed. Malformed files, non-array roots, and entries without an `address` are skipped rather than treated as an error. +This worker persists nothing. The broker owns a JSON array of `{ id, address, name?, tls_port?, mtls? }` at `configs/manual-nodes.json` beneath its shared application-data directory (§9). It persists successful add/remove operations atomically through a temporary file and rename, and replays entries as `node/add` whenever the worker starts. Entries without an address are skipped; unreadable or malformed configuration is reported rather than silently overwritten. The broker serializes configuration mutations and replay. It restores configuration, not cached proof or measurements, so each new worker must authenticate hardware again. ### 7.7 Error codes @@ -212,40 +213,40 @@ This worker persists nothing. The durable list is a JSON array of `{ id, address - **Upstream**: `nvpair-ui-broker` — the parent that spawns this worker (`--manual-nodes-path`), feeds it `node/add` / `node/remove` / `nodes/list` relayed from clients, and closes its `stdin` to stop it. The PAIR desktop app and `nvpair-tui` reach it only through that relay. - **Downstream**: `nvpair-ui-broker`'s discovery store, per-engine proxy bridge, and telemetry cache; `nvpair-errors` (via the broker) for probe-failed reports; `nvpair-proxy`, whose Ollama and LM Studio facades receive bridged manual nodes; `nvpair-job-scheduler`, which ranks them alongside discovered nodes using the GPU telemetry the broker relays. -- **External**: each probed host's Ollama server (`:11434`), LM Studio OpenAI-compatible server (`:1234`), and `nvpair-node-info` endpoint (`:14318` plain, or the entry's `tls_port` over TLS); the cluster directory minted by `nvpair-cluster-manager` when `--cluster-dir` is passed; `github.com/Microsoft/go-winio` for the Windows named-pipe transport. +- **External**: each probed host's Ollama server (`:11434`), LM Studio OpenAI-compatible server (`:1234`), and `nvpair-node-info` endpoint (`:14318` or the entry's `tls_port`, always pinned mTLS); the cluster directory minted by `nvpair-cluster-manager` when `--cluster-dir` is passed; `github.com/Microsoft/go-winio` for the Windows named-pipe transport. ## 9. Data Ownership - **Owned**: the in-memory `map[string]*trackedNode` — each entry's `ManualEntry`, its latest `ManualNodeStatus`, and its `consecutiveFails` counter. Nothing durable. -- **Source of truth**: authoritative for *current* manual-node reachability, per-engine model lists, and node-info telemetry while the process lives. Not the source of truth for the entry list — the desktop app's `manual-nodes.json` is, and the user is. Not the source of truth for `hostUuid`, which the remote's `nvpair-node-info` reports. -- **Storage**: none written by this binary. The durable list is `configs/manual-nodes.json` under the per-user PAIR data directory: `%LOCALAPPDATA%\Nvidia Corporation\Personal AI Router\configs\manual-nodes.json` on Windows, `$XDG_CONFIG_HOME/Nvidia Corporation/Personal AI Router/configs/manual-nodes.json` (falling back to `~/.config`) on Linux, and `~/Library/Application Support/Nvidia Corporation/Personal AI Router/configs/manual-nodes.json` on macOS. `services/shared/appdir` resolves that same `/Nvidia Corporation/Personal AI Router` root for Go components, and the broker's default `--cluster-dir` is `cluster/` beneath it; this worker imports neither and only ever reads the cluster directory it is handed. +- **Source of truth**: authoritative for *current* manual-node reachability, per-engine model lists, and authenticated node-info telemetry while the process lives. The broker owns the durable entry list and the user chooses its contents. Remote `hostUuid` is accepted only when it matches the certificate and current pin. +- **Storage**: none written by this binary. Broker owns `configs/manual-nodes.json` under the shared per-user PAIR data root, or its configured manual-node file. `services/shared/appdir` resolves the default application-data directory; this worker reads only the supplied cluster directory and keeps configurations and probe results in memory. ## 10. Design Constraints -- **Performance**: probe cost is bounded by `probeTimeout` (3 s) per request, and entries are probed sequentially within a cycle, so a cycle over N nodes can take up to roughly 4 × 3 s × N in the worst case — with the fixed 10 s ticker, unreachable nodes can push a cycle past its interval. Keep-alives are disabled by design; a fresh dial per probe is cheap at this cadence and guarantees re-resolution. +- **Performance**: probe cost is bounded by `probeTimeout` (3 s) per request, and entries are probed sequentially within a cycle, so a cycle over N nodes can take up to roughly 4 × 3 s × N in the worst case — with the fixed 10 s ticker, unreachable nodes can push a cycle past its interval. All probes use fresh connections to re-resolve manual hostnames. Known peer hardware reads reuse bounded cluster clients but close each connection after the response. - **Scalability**: sized for a handful of hand-entered nodes on a small cluster, not for fleet-scale scanning. State is one map entry per node. -- **Reliability**: probes are best-effort with no retry inside a cycle; the next cycle is the retry. A three-cycle grace window absorbs transient failures before the user sees an error. Restarting the worker loses all entries; the broker evicts the orphans so no stale node lingers. -- **Security**: no listening socket, so no inbound attack surface. A manually entered address is a user assertion, not proof of a trusted peer, and a successful plain-HTTP or operator-TLS probe authenticates nothing. Cluster identity, pins, pairing, and membership belong to `nvpair-cluster-manager`; this worker only consumes the mesh read-only to build a client config. Certificate and key material is read from the paths given on the command line and never logged or emitted. +- **Reliability**: probes are best-effort with no retry inside a cycle; the next cycle is the retry. A three-cycle grace window absorbs transient failures before the user sees an error. The broker withdraws retired probe state and replays durable configurations after worker restart. +- **Security**: no listening socket. Manual addresses authorize explicit engine routes, never public hardware as a service input. Hardware requires exact current cluster pins and matching response identities. Cluster identity, pairing, pins, and membership remain owned by `nvpair-cluster-manager`; this worker consumes the mesh read-only and never logs key material. - **Compliance**: probe traffic touches only capability and telemetry endpoints. No prompt, chat message, request body, or inference response is read, forwarded, or logged. ## 11. Assumptions - `nvpair-ui-broker` is the parent: it spawns this worker, relays client requests to it, and closing its `stdin` is the shutdown signal. - The entered address is a bare host or IP, with no port and no scheme; the worker appends its own ports. -- Remote engines listen on their defaults (`11434`, `1234`), and remote `nvpair-node-info` on `14318` for plain HTTP. -- A clustered peer's node-info is pin-gated mTLS with no plaintext listener, so reaching it requires `tls_port` plus a populated cluster directory. +- Remote engines listen on their defaults (`11434`, `1234`), and remote `nvpair-node-info` on `14318` for pinned mTLS. +- Node-info supports public display HTTP and pinned mTLS on the registered port. Service hardware reads always require the latter. - `hostUuid` reported by a remote is stable for that host and unique in the fleet, which is what makes UUID-keyed deduplication with mDNS-discovered nodes correct. -- A client re-adds its manual nodes after this worker restarts. +- The broker replays saved manual entries after this worker restarts. ## 12. Failure Modes and Mitigations -- **Worker restart or crash**: every entry is lost, since state is memory-only. → `nvpair-ui-broker`'s `clearManualNodesState` drops the worker handle, evicts every manual-origin node from the discovery store (leaving a co-located scanner record intact), drops its manual telemetry, and withdraws it from the Ollama and LM Studio facades, so nothing stale is routed to; the client re-adds. Losing entries on restart is a known limitation with a fix in progress (§4). +- **Worker restart or crash**: probe state is lost. → The broker drops the retired handle, rejects its callbacks, removes manual-origin records and samples while preserving an independent scanner source, and withdraws manual proxy claims. It replays durable entries to the new worker, which authenticates hardware again. - **Host unreachable for three consecutive cycles**: the node reads down everywhere and inference cannot route to it. → One `errors:report` under `manual-nodes:probe-failed:` surfaces it in the UI after roughly 30 s, with re-add guidance for IP-literal entries; a reachable probe emits `errors:clear`. - **Entered address is a typo or a `host:port` string**: the probe URL is unreachable or malformed and the node reads permanently down. → The same probe-failed report fires; the address is not otherwise validated, so the operator has to correct the entry. -- **node-info drops while an engine stays up**: a blanked `hostUuid` would rekey the broker's discovery entry from UUID to manual id and back. → The last-learned `hostUuid` is carried forward whenever a failed node-info probe reports none. +- **node-info drops while an engine stays up**: the explicit engine route survives under its configuration key, while verification and service measurements become unavailable. - **Remote returns `200` with an unparseable model list**: a strict parse would report a live engine as down. → The engine is reported up with an empty model list; only the models are lost. -- **Remove races the initial probe**: a removed node could be resurrected by an in-flight probe's `node/discovered`. → Both `addNode`'s probe goroutine and `probeNode` re-check the map under lock after probing and drop the result when the id is gone. -- **TLS material misconfigured** (`--client-cert` without `--client-key`, unreadable bundle, no PEM in the bundle): probes would silently fall back to an unintended transport. → `tlsClientOptions.validate()` and `buildTLSClient` fail at startup with a fatal log rather than starting degraded. +- **Remove or replacement races a probe**: a removed entry could be resurrected, or a replacement under the same ID could inherit an old response. → Both the probe and initial-discovery publication require the original tracked entry to remain current under the manager lock. +- **Cluster identity or pin unavailable**: service hardware cannot authenticate. → Hardware remains unavailable or is withdrawn; no operator CA, unencrypted request, or configured engine supplies substitute proof. Explicit engine routes remain independent. - **Malformed inbound frame**: a bad params blob could panic a handler or corrupt state. → Every handler unmarshals into a typed struct and answers `-32602` on failure; unknown methods answer `-32601`; notifications are logged and dropped. - **Parent pipe severed** (`stdin` EOF): an orphaned prober would keep probing with nowhere to report. → EOF returns from `readLoop`, which cancels the context, stops the probe loop, and exits zero. `SIGINT`/`SIGTERM` and a `shutdown` request take the same path. @@ -259,9 +260,9 @@ This worker persists nothing. The durable list is a JSON array of `{ id, address `nvpair-ui-broker` spawns the worker as `nvpair-manual-nodes --log-level info --cluster-dir ` over stdio pipes. The worker emits `ready` carrying the version stamped into the binary at build time, which the broker logs, then starts its 10-second probe ticker. -A user enters `10.0.1.50` and names it `lab`. The client's request reaches the broker, which relays it with `RelayRequest` (no timeout, because `node/add` triggers a probe) as `{"jsonrpc":"2.0","id":1,"method":"node/add","params":{"address":"10.0.1.50","name":"lab"}}`. The worker computes `nodeID` = `"lab"`, stores the entry, and answers immediately with an unprobed `ManualNodeStatus` — `ollama_port` `11434`, `lmstudio_port` `1234`, `node_info_port` `14318`, every `*_up` false — which the broker relays back so the UI can render the row at once. +A user enters `10.0.1.50` and names it `lab`. The broker relays `{"jsonrpc":"2.0","id":1,"method":"node/add","params":{"address":"10.0.1.50","name":"lab"}}`. The worker stores entry `lab` and immediately returns unprobed status: engine and node-info ports filled, every `*_up` false, and unverified identity. Broker persists the successful entry and returns the result to the client while the initial probe runs independently. -On its goroutine the worker probes: `GET http://10.0.1.50:11434/` returns `200`, so `GET /api/tags` yields `["llama3.2:latest"]`; `GET http://10.0.1.50:1234/v1/models` fails to connect, so LM Studio is down; `GET http://10.0.1.50:14318/v1/node-info` returns one GPU, a CPU object, a memory object, a valid GPU sample with its age, and `hostUuid`. It emits `node/discovered` with the filled `ManualNodeStatus`. The broker's `forwardManualNodesNotification` upserts it through `manualToEnriched` into the discovery store keyed by the reported `hostUuid`, caches its GPU telemetry for the scheduler under the same key, and `bridgeManualNode` sends `nvpair-proxy` `ollama:node/add-manual` under that key and `lmstudio:node/remove-manual`. The node now appears in `discovery:nodes-changed` beside mDNS nodes and is a routable, schedulable Ollama target. +On its goroutine the worker probes: `GET http://10.0.1.50:11434/` returns `200`, so `GET /api/tags` yields `["llama3.2:latest"]`; `GET http://10.0.1.50:1234/v1/models` fails to connect, so LM Studio is down; `GET https://10.0.1.50:14318/v1/node-info` over pinned mTLS returns one GPU, a CPU object, a memory object, a valid GPU sample with its age, and matching host/cluster identities. Without a valid pin, hardware stays unavailable while explicit engine routing remains supported. It emits `node/discovered` with the filled `ManualNodeStatus`. The broker's `forwardManualNodesNotification` upserts it through `manualToEnriched` into the discovery store keyed by the reported `hostUuid`, caches its GPU telemetry for the scheduler under the same key, and `bridgeManualNode` sends `nvpair-proxy` `ollama:node/add-manual` under that key and `lmstudio:node/remove-manual`. The node now appears in `discovery:nodes-changed` beside mDNS nodes and is a routable, schedulable Ollama target. The host is then powered off. The first cycle that finds nothing emits `node/updated` with every `*_up` false, which withdraws the proxy candidate. After three consecutive such cycles the worker emits one `errors:report` with `id` `manual-nodes:probe-failed:lab` and `severity` `"warning"`; the broker stamps the local node id and timestamp and forwards it to `nvpair-errors`. When the host returns, the next probe emits `node/updated` and `errors:clear` for the same id. If the user instead removes the node, `node/remove` answers `{"removed": true}`, emits `node/removed` with `{"id":"lab"}` and an `errors:clear`, and the broker releases the store and proxy claims. @@ -272,14 +273,11 @@ The host is then powered off. The first cycle that finds nothing emits `node/upd | Flag | Default | Effect | |---|---|---| | `--ipc ` | `""` (stdio) | Dial a Unix domain socket or Windows named pipe via `nvpair-shared/ipc` instead of using `stdin`/`stdout`. A dial failure is fatal. | -| `--client-cert ` | `""` | PEM client certificate presented on every TLS probe. Requires `--client-key`. | -| `--client-key ` | `""` | PEM private key matching `--client-cert`. Requires `--client-cert`. | -| `--ca-bundle ` | `""` | PEM CA bundle for verifying probe server certificates, appended to the system trust store. | -| `--cluster-dir ` | `""` | Cluster config dir (`node.crt` / `node.key` plus `trusted/`). When it resolves to a clustered mesh, a `tls_port` probe uses cluster mTLS. | +| `--cluster-dir ` | `""` | Cluster config dir (`node.crt` / `node.key` plus `trusted/`). All hardware reads require current cluster membership and pinned mTLS. | | `--log-level ` | `$NVPAIR_LOG_LEVEL`, else `info` | Initial `slog` level (`debug`\|`info`\|`warn`\|`error`); registered by `applog.RegisterFlag`. | | `--version` | off | Print `Version` and exit `0`. | -Startup order: parse flags, print version and exit if asked, `applog.Init`, `clustertrust.Open(--cluster-dir)` (a live view, so a cluster joined later is picked up without a restart), validate the TLS option triple (fatal on a half-configured pair), open the transport, install the `SIGINT`/`SIGTERM` handler, construct the `Manager`, emit `ready`, start the probe loop, and enter the read loop. +Startup order: parse flags, print version and exit if asked, `applog.Init`, `clustertrust.Open(--cluster-dir)` (a live view, so a cluster joined later is picked up without a restart), open the transport, install the `SIGINT`/`SIGTERM` handler, construct the `Manager`, emit `ready`, start the probe loop, and enter the read loop. The process is always a child. `nvpair-ui-broker` spawns the binary named by its `--manual-nodes-path` flag, which the desktop application always passes, and otherwise looks for `nvpair-manual-nodes` in its own working directory. It spawns it through `startRPCWorker` with `--log-level ` plus `--cluster-dir ` when the broker resolved one, wires `stdin`/`stdout` pipes, and inherits the child's `stderr`. It is registered as an optional worker under a restart policy, so a crash respawns it and triggers `clearManualNodesState`. The broker's `Stop` closes `stdin`; the child observes EOF, returns from `readLoop`, cancels its context to stop the probe loop, closes the transport, and exits `0`. A `shutdown` request and a caught signal follow the same cancellation path. Electron never launches this worker directly — only the broker does. diff --git a/services/nvpair-node-info/spec.md b/services/nvpair-node-info/spec.md index 826bb4f62..cb0599191 100644 --- a/services/nvpair-node-info/spec.md +++ b/services/nvpair-node-info/spec.md @@ -6,20 +6,21 @@ SPDX-License-Identifier: Apache-2.0 # Microservice: Node Info (`nvpair-node-info`) ## 1. Purpose -The per-node hardware reporter. It detects this host's GPU, CPU, and physical-memory inventory at startup, samples live utilization on a one-second tick, and serves the merged result as one JSON document at `GET /v1/node-info` on TCP `14318`. It is the only place in the fleet where a machine's own accelerator and load numbers are produced; every node card, GPU chart, and capacity readout anywhere in the product is downstream of this route. +The per-node hardware reporter. It detects this host's GPU, CPU, and physical-memory inventory at startup, samples live utilization on a one-second tick, and serves the merged result as one JSON document at `GET /v1/node-info` on TCP `14318`. It also publishes private inventory snapshots to its owning broker. Public HTTP supplies frontend displays; services use authenticated remote reads or locally owned IPC measurements. -It exists as a separate binary because the readout is inherently platform-specific and privileged-adjacent — DXGI COM calls and PDH performance counters on Windows, `/proc` plus `nvidia-smi` on Linux, `ioreg` plus Mach counters on macOS, `ghw` for CPU identity everywhere — and because its consumers are heterogeneous: `nvpair-node-scanner` fetches it to enrich the node directory, `nvpair-manual-nodes` probes it to identify a typed-in address, the Electron desktop polls each discovered node's advertised port directly every two seconds, and `nvpair-tui` polls the one node whose detail screen is open. It carries no state beyond the current sample and answers exactly one route. It makes no routing decision itself, but its GPU sample is the input `nvpair-job-scheduler` turns into GPU pressure, by way of the scanner and `nvpair-manual-nodes`. It also reports upward which of this host's addresses remote peers have actually reached it on, which the scanner uses to rank the addresses it publishes. +It exists as a separate binary because the readout is inherently platform-specific and privileged-adjacent — DXGI COM calls and PDH performance counters on Windows, `/proc` plus `nvidia-smi` on Linux, `ioreg` plus Mach counters on macOS, `ghw` for CPU identity everywhere — and because its consumers are heterogeneous: remote scanner and manual-node workers read protected inventory over pinned mTLS, the local scanner receives owned IPC snapshots through the broker, and desktop and TUI poll public HTTP for display. It keeps the current sample, local registry metadata, and address observations in memory and makes no routing decision. Its accepted GPU samples reach scheduler through scanner or manual-node workers and the broker. It also reports which of this host's own addresses served remote requests, which scanner uses as reachability evidence. ## 2. Scope **In scope** - Static hardware detection at startup: GPU adapters (name + total VRAM), CPU (model + physical core count), total physical RAM. - Live utilization sampling on a one-second cadence: per-GPU VRAM used and busy percent, node CPU busy percent, memory used. - Reporting whether a usable GPU utilization sample exists and how old it is (`telemetryValid`, `msSince`), so consumers can tell an idle 0 % reading from no reading and a fresh sample from a stale one. -- Serving the merged inventory as JSON over HTTP at `/v1/node-info`. +- Serving public display inventory over HTTP and protected peer inventory over pinned mTLS at `/v1/node-info` on the same registered port. +- Publishing local hardware snapshots over private broker/worker IPC before membership and after departure. +- Receiving owned local registry metadata over IPC and exposing it only in authenticated peer inventory. - Recording which local addresses served non-loopback peers and reporting that set to the parent as `nodeinfo:observed-addresses`. -- Reporting this node's stable per-host UUID (`hostUuid`) so an HTTP-only consumer can key the node by permanent identity. -- Reporting the cluster principal this node holds (`clusterUuid`) so a peer can learn its membership without receiving its mDNS record. -- Optional bring-your-own TLS/mTLS listener, and an optional cluster-gated listener whose personality follows live cluster membership. +- Reporting `hostUuid` and `clusterUuid`; public values are display claims and cannot establish identity or membership. +- Optional bring-your-own TLS/mTLS listener, and a cluster listener whose protected handler follows live membership while public HTTP remains available. **Out of scope** - mDNS advertisement and node discovery — `nvpair-node-scanner` owns the single `_nvpair-node` record and registers this service's port as `ni`. @@ -30,33 +31,33 @@ It exists as a separate binary because the readout is inherently platform-specif - Persisting telemetry history, aggregating across nodes, or raising alerts — consumers do that; this service reports only the current sample. ## 3. Key Use Cases -- **Local enrichment**: the node-scanner daemon reads its own node over `127.0.0.1:14318/v1/node-info` and folds `GPUs` / `cpu` / `memory` into this node's `DirectoryNode` before publishing the record. -- **Peer enrichment**: a peer's daemon fetches `/v1/node-info` at the `ni=` port from the neighbor's `_nvpair-node` TXT, over plain HTTP by policy (`noderec.ServiceNodeInfo.Transport()` is `TransportPlain` even when the target node is clustered). -- **Desktop telemetry poll**: the Electron main process polls every discovered node's `http://:/v1/node-info` every 2000 ms with a 1500 ms per-attempt timeout, and merges the response into its node model. Repeated failures back off to a 30 s cap; for a multi-homed node it remembers the address that answered and asks only that one; and it reads its own machine over `127.0.0.1` rather than its LAN address. This is the desktop's one sanctioned direct-HTTP exception to going through the broker's JSON-RPC surface, because the broker exposes no rich per-node telemetry. -- **Terminal telemetry poll**: `nvpair-tui` makes the same exception for the same reason, but polls only the node whose detail screen is open, at the desktop's cadence (2000 ms, 1500 ms timeout). It reads its own machine over `127.0.0.1`, tries a multi-homed node's addresses starting with the one that last answered, discards an answer whose `hostUuid` names another machine, caps the body at 256 KiB, and refuses redirects. It never polls over TLS: for a hand-added node that serves node-info only over TLS it shows the readings from that node's `nvpair-manual-nodes` probe instead. -- **Scheduler GPU pressure**: the scanner daemon and `nvpair-manual-nodes` each read `telemetryValid`, `msSince`, and the highest `utilization_percent` across `GPUs`, and hand that compact sample to the broker, which ages it and relays it to `nvpair-job-scheduler`. A sample with `telemetryValid: false`, or one that has aged past the scheduler's freshness window, counts as neutral pressure. -- **Peer-observed addresses**: every request served to a non-loopback peer records the local address it arrived on. Every 30 s the service sends the parent `nodeinfo:observed-addresses` with the current set; the broker relays it to the scanner daemon as `discovery:set-observed-addresses`, which ranks a peer-proven address above one it only inferred. -- **Manual node identification**: `nvpair-manual-nodes` probes a user-typed address on `14318` (or the entry's `TLSPort` over HTTPS) every ten seconds and reads `hostUuid` so a node added by address is tracked under the same identity the rest of the fleet keys on. -- **Membership convergence**: a peer's node-scanner daemon re-reads `clusterUuid` from every directory entry's `ni` port on its fifteen-second sweep, because membership otherwise reaches the fleet only as the `cluster-uuid=` TXT key on a record whose change it may never receive. A failed fetch, an absent field, or an answer whose `hostUuid` disagrees with the entry all leave the peer's annotation untouched; only an explicit, identity-matched report is acted on. +- **Local enrichment**: startup and two-second `nodeinfo:info-updated` snapshots travel through the broker as `discovery:local-node-info`. Scanner attaches them to its owned registry record, without a loopback HTTP service read. +- **Peer enrichment**: a remote scanner authenticates the node-info endpoint over exact-pin mTLS before promoting an announcement. Protected metadata supplies the peer's name, addresses, service ports, and hardware. Failed authentication never triggers public HTTP fallback. +- **Desktop telemetry poll**: Electron polls public HTTP every 2000 ms with a 1500 ms per-attempt timeout, existing address failover, and failure backoff capped at 30 s. It updates hardware display fields only, independently of verified broker snapshots. It reads this host over loopback. Public responses cannot change identity, trust, addresses, service ports, or models. +- **Terminal telemetry poll**: TUI polls the open node detail screen over public HTTP at the same cadence and timeout. It retains its response-size limit and refuses redirects. Verified rows require a matching response `hostUuid`; an unverified row's display preview does not turn the response's UUID into verified identity. +- **Scheduler GPU pressure**: scanner uses owned IPC locally and pinned mTLS remotely; manual-node hardware also requires pinned mTLS. They supply maximum GPU utilization with validity and elapsed age through the broker. Unknown or stale data follows the scheduler's existing freshness and pressure rules, and never becomes freshly idle through a failed read or replay. +- **Observed local addresses**: a non-loopback request records the real local address it reached. Every 30 s the service sends the current set as `nodeinfo:observed-addresses`; broker relays it to scanner. This is reachability evidence, including public requests, and does not authenticate the requester. +- **Manual node identification**: the manual worker probes `14318`, or the configured node-info port, over pinned mTLS every ten seconds. Matching response host/principal and certificate identities allow the broker to join that source to the same verified host. Explicit engine routes remain independent of hardware authentication. +- **Membership convergence**: live cluster state gates every protected request, including reused connections. Departure or pin replacement revokes authorization; public inventory remains available. Protected responses return `503` until owned metadata agrees with the serving node's current identity. - **Edge case — no GPU**: `detectGPUs` returns an empty slice; `buildResponse` still allocates the slice, so the body is `{"GPUs":[],"telemetryValid":false,"msSince":0}` plus whatever CPU/memory data exists. The route never 404s or errors for a GPU-less host. - **Edge case — GPU collection stops working**: a failed pass keeps the last usable sample and its original timestamp, so `telemetryValid` stays `true` and `msSince` keeps growing. Consumers read the growing age as staleness rather than seeing numbers blank out or silently freeze. -- **Edge case — unmatched adapter**: a GPU whose `statsKey` is absent from the current snapshot keeps zero-valued dynamic fields, which `omitempty` drops. Clients see a missing `vram_used_bytes` rather than a misleading literal `0` (`TestBuildResponseOmitsZero`). -- **Edge case — static introspection fails**: a nil `*CPUInfo` or a zero `memTotal` drops the whole `cpu` / `memory` object, never an empty shell like `"cpu":{}` (`TestBuildResponseCPUMemoryMatrix`). -- **Edge case — unified memory (UMA)**: on Grace-Blackwell class hosts `nvidia-smi` returns `[N/A]` for `memory.total` / `memory.used`; `vram_bytes` is backfilled from total system RAM and `vram_used_bytes` from the `/proc/meminfo` used figure, which keeps working even when dynamic `nvidia-smi` collection never succeeds (`TestBuildResponseUnifiedMemoryUsesSystemSnapshot`). -- **Edge case — macOS enumeration fails at startup**: `detectGPUs` returns nothing, but the collector re-reads IORegistry every tick and `mergeGPUInventory` folds the recovered adapters into the response, so the inventory repairs itself without a restart (`TestBuildResponseRecoversDarwinGPUInventory`). -- **Edge case — RDP session on Windows**: DXGI enumerates a phantom clone of the physical adapter plus Microsoft's remoting adapters. Both a name denylist and a `HKLM\SOFTWARE\Microsoft\DirectX` `AdapterLuid` gate suppress them (`TestIsVirtualDisplayAdapter`, `TestKeepPhysicalAdapter`). -- **Edge case — clustered read (`--cluster-dir` only)**: a pinned peer gets `200`; the node reading itself is accepted through self-trust; a peer that completes the handshake but is not pinned here gets `403` (`TestNodeInfoHandler_MTLSGate`). +- **Edge case — unmatched adapter**: a GPU whose `statsKey` is absent from the current snapshot keeps zero-valued dynamic fields, which `omitempty` drops. Clients see a missing `vram_used_bytes` rather than a misleading literal `0`. +- **Edge case — static introspection fails**: a nil `*CPUInfo` or a zero `memTotal` drops the whole `cpu` / `memory` object, never an empty shell like `"cpu":{}`. +- **Edge case — unified memory (UMA)**: on Grace-Blackwell class hosts `nvidia-smi` returns `[N/A]` for `memory.total` / `memory.used`; `vram_bytes` is backfilled from total system RAM and `vram_used_bytes` from the `/proc/meminfo` used figure, which keeps working even when dynamic `nvidia-smi` collection never succeeds. +- **Edge case — macOS enumeration fails at startup**: `detectGPUs` returns nothing, but the collector re-reads IORegistry every tick and `mergeGPUInventory` folds the recovered adapters into the response, so the inventory repairs itself without a restart. +- **Edge case — RDP session on Windows**: DXGI enumerates a phantom clone of the physical adapter plus Microsoft's remoting adapters. Both a name denylist and a `HKLM\SOFTWARE\Microsoft\DirectX` `AdapterLuid` gate suppress them. +- **Edge case — clustered read (`--cluster-dir` only)**: a pinned peer with matching owned metadata gets `200`; an unpinned client gets `403`. Shared TLS self-trust remains supported, while local service measurements use IPC. ## 4. Open Questions / Risks -- **Risk — plaintext LAN inventory**: under the broker, `nvpair-node-info` is deliberately spawned *without* `--cluster-dir`, so the GPU/CPU/memory inventory is readable by anything on the local subnet. It is the one documented exception to "the inter-node cluster data plane is always mTLS", accepted because the desktop's two-second poll and baseline discovery hold no cluster identity. The cluster-gated mode is implemented and unit-tested but unused in the shipped topology; enabling it without first solving the desktop read blanks every node card. +- **Public LAN inventory**: public HTTP intentionally exposes hardware measurements for desktop and TUI, including while clustered. Those responses cannot authenticate the endpoint or supply service decisions. Broker supplies `--cluster-dir` so pinned mTLS is independently available on the same port. - **Risk — one-shot hardware detection**: `detectGPUs`, `detectCPU`, and `detectMemoryTotal` run once in `main`, so the CPU and memory readout is fixed for the process lifetime and a Windows or Linux adapter added, removed, or driver-installed after startup never appears. macOS is the exception: its collector re-enumerates IORegistry every tick and `mergeGPUInventory` adds adapters the startup pass did not see. - **Risk — latched telemetry unavailability**: Linux latches `nvidiaUnavailable` on the first `nvidia-smi` failure and never retries; Windows adds PDH counters only inside `startStatsCollector`. A driver or counter set that becomes available later is not picked up for the process lifetime. macOS retries `ioreg` on every tick instead. - **Risk — a stopped collector keeps serving its last GPU numbers**: on every platform a failed GPU pass retains the last usable sample, so a host whose `nvidia-smi` latched, whose PDH query broke, or whose IORegistry read started failing keeps reporting the old VRAM and utilization figures indefinitely. `msSince` growing is the only signal. The scheduler path honors it; a consumer that renders `utilization_percent` without checking `msSince` shows a frozen number. - **Risk — `--tls-port` default collides**: `14319` is also the broker's fixed `nvpair-errors` port. Nothing binds it today because the broker never passes `--cert`/`--key`, and the cluster-gated listener deliberately reuses `--port` instead. Any future use of the bring-your-own TLS path under the broker must move this port. - **Risk — unified-memory VRAM is system RAM**: on a unified-memory host `vram_bytes` is the total physical system memory — on Linux `vram_used_bytes` is the whole-system used figure as well, while on Apple Silicon it is the GPU driver's mapped allocation (`Alloc system memory`) rather than either the active working set or system-wide usage — so a consumer that sums "VRAM + RAM" double-counts. Nothing on the wire marks a GPU as unified. - **Risk — undocumented IORegistry keys**: the macOS GPU numbers come from the `PerformanceStatistics` dictionary of an `IOAccelerator` entry (`Device Utilization %`, `GPU Activity(%)`, `Alloc system memory`, `vramUsedBytes`, `inUseVidMemoryBytes`, `vramFreeBytes`), none of which Apple documents or guarantees. A renamed memory key drops that one metric to zero, where `omitempty` hides it. A renamed utilization key is more visible: a pass with no utilization reading is not a usable sample, so `telemetryValid` stays `false` (or the previous sample ages). Either way a macOS release can degrade the readout without surfacing an error. -- **Risk — macOS VRAM used without a total**: an adapter that exposes neither `VRAM,totalMB` nor a dedicated used counter still reports `Alloc system memory` as `vram_used_bytes`, so a response can carry a used figure with no `vram_bytes` to divide it by (`TestParseIORegistryGPUsPreservesDedicatedCounterPresence`). A consumer rendering a VRAM percentage has to treat the absent total as "no percentage" rather than zero. -- **Risk — the reported cluster principal rides one best-effort push**: under the broker `clusterUuid` comes only from `nodeinfo:set-cluster-identity`, written on spawn and on every membership or pin-set change. A failed write is warned about and not retried, so a dropped frame leaves peers reading a stale principal — or none at all — until the next change or a restart. +- **Risk — macOS VRAM used without a total**: an adapter that exposes neither `VRAM,totalMB` nor a dedicated used counter still reports `Alloc system memory` as `vram_used_bytes`, so a response can carry a used figure with no `vram_bytes` to divide it by. A consumer rendering a VRAM percentage has to treat the absent total as "no percentage" rather than zero. +- **Registry metadata is initially unavailable**: the broker supplies complete local metadata over IPC and resends it after worker restart. Protected reads return `503` while it is absent or disagrees with current identity. Public display and owned local measurements remain independent. - **Risk — fatal listener errors**: a bind failure or a non-`ErrServerClosed` serve error calls `log.Fatalf`, terminating the process. Recovery is the broker's supervisor restart, not in-process. - **`inference_hardware_ids` (open — needs a producer if the filter is ever to engage)**: the desktop's merge reads an optional `inference_hardware_ids` string array from the response, but no field with that JSON tag exists in `NodeInfoResponse` or anywhere under `services/`, so nothing produces it today. This is a forward-compatibility hook rather than a broken read: the consumer preserves the absent-versus-empty distinction deliberately — absent means "no readiness reported, show every GPU" and `[]` means "no inference-ready hardware" — so the unproduced field degrades to showing all GPUs. The filter is display-only and never influences routing, which stays owned by `nvpair-proxy` and `nvpair-job-scheduler`. Specifying and producing the field here is what would make it engage; until then the response schema in §7.1 is complete as written. - **Firewall rule surface (open — needs an installer audit)**: `nvpair-setup.nsi` grants this binary an inbound UDP 5353 mDNS rule (`NVPAIR mDNS Node Info`) in addition to the TCP rule it needs. The process opens no multicast socket — discovery belongs to the node-scanner daemon — so that rule widens the binary's inbound surface for no functional reason. @@ -69,15 +70,18 @@ It exists as a separate binary because the readout is inherently platform-specif - Detect GPU adapters, CPU identity, and total physical memory at startup, per OS, degrading to fewer fields rather than failing. - Sample dynamic utilization on a background ticker and publish each pass atomically as one `statsSnapshot`. - Merge static identity with the latest snapshot per request and serve it as `application/json` at `/v1/node-info`. +- Publish `nodeinfo:info-updated {info}` over owned stdio at startup and every two seconds. Broker forwarding and replay preserve validity and source age plus elapsed in-process time; time buffered in worker stdio before receipt is an accepted approximation. +- Accept `nodeinfo:set-peer-metadata {hostUuid, clusterUuid, peer}` from the owning broker, replace the complete snapshot, and require it to match current local identity before serving protected inventory. +- Keep public HTTP available in every membership state. Service reads require exact-current-pin mTLS remotely and owned IPC locally; public data cannot supply scheduling, routing, availability, or verified metadata. - Mark a GPU pass usable only when at least one adapter yielded a utilization reading (a parsed 0 % counts); report `telemetryValid` and the usable sample's age as `msSince`, and keep the previous usable sample, timestamp included, when a pass fails. - Record the local address of every request served to a non-loopback peer, expire each after 5 minutes unseen, and send the full current set, empty included, as `nodeinfo:observed-addresses` every 30 s. - Report `hostUuid`, taking `--node-id` verbatim when supplied and otherwise resolving the shared local identity. - Report `clusterUuid` from the live trust store when cluster-gated and from the parent's push otherwise, keeping "unknown" distinct from "belongs to no cluster". - Honor the bring-your-own TLS contract (`--cert`/`--key`/`--client-ca`) and the cluster-gated contract (`--cluster-dir`) with the documented precedence. -- Handle `log/set-level` and `nodeinfo:set-cluster-identity` on stdin and shut down on stdin EOF, `SIGINT`, or `SIGTERM`. +- Handle `log/set-level`, `nodeinfo:set-cluster-identity`, and `nodeinfo:set-peer-metadata` on stdin and shut down on stdin EOF, `SIGINT`, or `SIGTERM`. **Non-functional** -- The request path performs no hardware sampling and takes no lock: one atomic pointer load, the JSON marshal, and — only when cluster-gated — a cluster-dir refresh. It must stay well inside a 1500 ms client timeout under a two-second poll from every peer. +- The request path performs no hardware sampling. It reads the atomic collector snapshot and serializes it; cluster mode also refreshes live trust, and protected reads take short metadata locks. It must stay well inside the existing client timeout under two-second polling. - Compiles and runs on Windows, Linux, and macOS × amd64/arm64; the telemetry paths use direct syscalls, file reads, purego bindings, and short-lived unprivileged helper processes rather than cgo. - Unknown values are omitted from JSON, never reported as a literal zero. - Never logs prompts, request bodies, certificates, or key material. @@ -87,7 +91,7 @@ It exists as a separate binary because the readout is inherently platform-specif **Inputs** — CLI flags at startup (§15); the host's own hardware interfaces (DXGI, PDH, `GlobalMemoryStatusEx`, `/proc/stat`, `/proc/meminfo`, `nvidia-smi`, `/usr/sbin/ioreg`, Mach through gopsutil, `ghw`); optionally a cluster dir (`node.crt`, `node.key`, `trusted/*.json`, `admission.json`) read through `nvpair-shared/clustertrust`; the identity store read through `nvpair-shared/nodeid`; and newline-delimited JSON-RPC 2.0 control frames on stdin. There is no request payload — `/v1/node-info` takes no parameters, headers, or query string. -**Outputs** — one JSON body per request, and stderr logs via `nvpair-shared/applog`. On stdout the service emits one notification, `nodeinfo:observed-addresses` every 30 s, and the response to an id-bearing `log/set-level` request; the two share one writer so their frames never interleave. It emits no `ready` frame. +**Outputs** — one JSON body per request, and stderr logs via `nvpair-shared/applog`. Serialized stdout carries `nodeinfo:info-updated` at startup and every two seconds, `nodeinfo:observed-addresses` every 30 s, and replies to id-bearing control requests. It emits no `ready` frame. `NodeInfoResponse` (top level): ```json @@ -140,14 +144,15 @@ Minimum response, from a host with no GPU, no `ghw` introspection, and no cluste ``` ## 7. API / Interface Contract -Two interfaces: an HTTP inventory route toward every consumer (the node-scanner daemon, `nvpair-manual-nodes`, the Electron desktop's direct poll, and `nvpair-tui`'s node detail screen), and a stdio JSON-RPC control channel toward the parent process that spawned it. +Two interfaces: the inventory route with independent public HTTP and protected mTLS handlers, and owned stdio JSON-RPC for local snapshots, metadata, and control. ### 7.0 Methods and notifications HTTP surface — one route, exactly: | Route | Method | Auth | Success | Failure | |-------|--------|------|---------|---------| -| `/v1/node-info` | any (the pattern is path-only, so `http.ServeMux` does not filter by method) | none by default; pinned-peer mTLS while `--cluster-dir` is set *and* this node is a cluster member | `200`, `Content-Type: application/json`, `NodeInfoResponse` | `403 forbidden: not a pinned cluster peer` | +| `/v1/node-info` over public HTTP | any (the pattern is path-only) | none, including while clustered | `200`, `Content-Type: application/json`, public `NodeInfoResponse` | — | +| `/v1/node-info` over cluster mTLS | any (the pattern is path-only) | live membership and the client's exact current pin | `200`, `NodeInfoResponse` plus protected `peer` metadata | handshake refused while unclustered, `403` for an unpinned client, `503` while local metadata is incomplete or inconsistent | | any other path | any | — | — | `404` from `http.ServeMux` | stdio surface — newline-delimited JSON-RPC 2.0 on stdin, handled by `applog.StdinRPC`: @@ -156,16 +161,18 @@ stdio surface — newline-delimited JSON-RPC 2.0 on stdin, handled by `applog.St |--------|--------|--------| | `log/set-level` | `{ level }` (`debug`\|`info`\|`warn`\|`warning`\|`error`) | `{ level }` when the frame carries an `id`; a rejected level answers `-32602` | | `nodeinfo:set-cluster-identity` | `{ clusterUuid }` (empty means "belongs to no cluster") | none — a notification; the value appears on the next `/v1/node-info` response | +| `nodeinfo:set-peer-metadata` | `{hostUuid, clusterUuid, peer: {name, addresses, services}}` | `{ok:true}` for an id-bearing request; malformed or mismatched host returns `-32602` | -`applog.StdinRPC` dispatches `log/set-level` itself and forwards every other frame to `handleClusterIdentity`, which ignores any method other than `nodeinfo:set-cluster-identity` and drops a malformed payload rather than latching a wrong membership. +`applog.StdinRPC` dispatches `log/set-level` itself. Identity and peer-metadata handlers accept their corresponding frames and ignore unrelated methods. A malformed identity push is dropped rather than changing the reported claim. With `--cluster-dir`, live trust-store state supplies `clusterUuid`; the identity push is used only without that directory. -Outbound, the service sends one notification to its parent: +Outbound, the service sends these notifications to its owning parent: | Notification | Params | Cadence | |--------------|--------|---------| | `nodeinfo:observed-addresses` | `{ addresses: string[] }` | every 30 s, unconditionally, including an empty list | +| `nodeinfo:info-updated` | `{info: NodeInfoResponse}` | startup and every two seconds | -It publishes nothing about inventory or telemetry; HTTP consumers poll. +Local snapshots have the public hardware shape and omit protected `peer` metadata. Broker forwards them privately to scanner; they are not frontend HTTP readings. ### 7.1 Response schema `buildResponse` copies the static `[]GPUInfo` detected at startup, folds in any adapters the snapshot's own `GPUInventory` carries (`mergeGPUInventory` enriches a matching `statsKey` in place and appends an unseen one), overlays the latest snapshot by `statsKey`, and marshals. Field semantics: @@ -188,8 +195,9 @@ It publishes nothing about inventory or telemetry; HTTP consumers poll. | `msSince` | `int64` | no — always present | age in ms of that sample at response time; `0` whenever `telemetryValid` is `false` | | `hostUuid` | `string` | yes | `resolveHostUUID()` | | `clusterUuid` | `*string` | yes — absent means the node does not know | `mesh.NodeUUID()` when cluster-gated, else the parent's `nodeinfo:set-cluster-identity` push | +| `peer` | `*noderec.PeerMetadata` | yes — protected response only | owned local registry snapshot: `{name, addresses, services}`, with service keys mapped to ports | -`GPUInfo.statsKey` (Windows adapter LUID, Linux GPU UUID, macOS IORegistry entry ID) and `GPUInfo.usesSystemMemoryUsage` are unexported and tagged `json:"-"`: the first joins the static record to the snapshot map, the second marks an adapter whose used bytes come from the system memory sample, and neither reaches the wire. An absent field means "unknown". A genuinely idle GPU also drops `utilization_percent`, and for display the two cases render identically. `telemetryValid` resolves that ambiguity at node level for anything that needs it: a valid sample whose highest reported utilization is 0 % is an idle node, not a missing reading. `telemetryValid` and `msSince` describe the node-wide GPU sample as a whole, not any one adapter, and consumers must ignore `msSince` while `telemetryValid` is `false`. `clusterUuid` is the one field where absent and empty are different answers — absent is "this node does not know its membership", empty is "this node belongs to no cluster" — and a consumer that collapses them clears a correct annotation elsewhere in the fleet, so it is a pointer rather than a plain `omitempty` string (`TestBuildResponseClusterUUIDWireStates`). +`GPUInfo.statsKey` (Windows adapter LUID, Linux GPU UUID, macOS IORegistry entry ID) and `GPUInfo.usesSystemMemoryUsage` are unexported and tagged `json:"-"`: the first joins the static record to the snapshot map, the second marks an adapter whose used bytes come from the system memory sample, and neither reaches the wire. An absent field means "unknown". A genuinely idle GPU also drops `utilization_percent`, and for display the two cases render identically. `telemetryValid` distinguishes a valid node-wide GPU sample whose highest utilization is 0 % from a missing reading. Consumers ignore `msSince` while validity is false. `clusterUuid` preserves three wire states: absent means unknown, empty claims unclustered, and a nonempty value claims that node principal. Public claims cannot change verified membership. ### 7.2 Per-OS hardware detection - **Windows GPU** (`gpu_windows.go`): `CreateDXGIFactory1` → `IDXGIFactory1::EnumAdapters1` → `IDXGIAdapter1::GetDesc1`, walked by hand-rolled COM vtable calls until `DXGI_ERROR_NOT_FOUND`. DXGI is the sole source because it is vendor-agnostic and returns name and VRAM in one struct; `Win32_VideoController.AdapterRAM` is explicitly rejected as a 32-bit field that is wrong for every modern dGPU. Adapters flagged `DXGI_ADAPTER_FLAG_SOFTWARE` are skipped, as are names matching `remote display`, `microsoft basic display`, or `virtual display adapter`, and any adapter whose LUID is absent from `HKLM\SOFTWARE\Microsoft\DirectX\*\AdapterLuid` (the gate is skipped entirely when that key is unreadable or empty). `statsKey` is `luidKey(low, high)` = `luid_0x%08x_0x%08x_phys_0`. @@ -210,20 +218,20 @@ Each snapshot also carries `GPUSampledAt`, the collection time of the latest **u - **Other** (`stats_other.go`, any platform without a collector): the collector is a no-op returning a zero snapshot, so every dynamic field is omitted. ### 7.4 Node identity -`resolveHostUUID(nodeID, clusterDir)` returns `--node-id` verbatim when non-empty. The broker always passes its own already-resolved node UUID, so `/v1/node-info` reports exactly the identity the fleet keys on even when the broker runs on a custom data root. Standalone, it calls `nodeid.Resolve(filepath.Dir(clusterDir))` — the empty string selects the shared per-user data dir — which prefers `/cluster/identity.json`'s `node_uuid` (the cluster-manager's principal), then `/node-id.json`, and otherwise mints and persists a random v4 UUID. Resolution never fails the caller. `TestResolveHostUUIDFlagWins` and `TestResolveHostUUIDFallsBackToClusterRoot` pin both branches. +`resolveHostUUID(nodeID, clusterDir)` returns `--node-id` verbatim when non-empty. The broker always passes its own already-resolved node UUID, so `/v1/node-info` reports exactly the identity the fleet keys on even when the broker runs on a custom data root. Standalone, it calls `nodeid.Resolve(filepath.Dir(clusterDir))` — the empty string selects the shared per-user data dir — which prefers `/cluster/identity.json`'s `node_uuid` (the cluster-manager's principal), then `/node-id.json`, and otherwise mints and persists a random v4 UUID. Resolution never fails the caller. -The cluster principal is resolved separately, and by deployment rather than by a fallback chain. With `--cluster-dir` this process holds the trust store and answers `mesh.NodeUUID()` while `mesh.Clustered()`, so the answer is always known. Without one it can read no membership at all — that is the price of keeping the inventory plain for every LAN peer — so it reports only what the parent pushed and stays absent until the first `nodeinfo:set-cluster-identity` frame arrives (`TestClusterIdentityStartsUnknown`). The two sources are mutually exclusive by construction. +The cluster principal is resolved separately by deployment mode. With `--cluster-dir`, including under the broker, the field is the live mesh principal while clustered and an empty string otherwise. Without that directory it reports the parent's identity push, remaining absent until the first push. Public HTTP carries an unverified claim in either mode; a consumer cannot use it to clear or establish membership. Protected responses require `hostUuid`, `clusterUuid`, and the owned registry identities to equal the serving certificate principal. ### 7.5 Listeners, TLS, and cluster trust Three mutually-exclusive listener layouts, decided at startup from the flags: -1. **Plaintext (default, and what the broker runs)**: one HTTP listener on `--port`. `--cluster-dir` unset, no cert. +1. **Plaintext standalone default**: one public HTTP listener on `--port`, with no cluster directory or operator cert. 2. **Bring-your-own TLS**: `--cert` + `--key` build a `tls.Config` pinned to `MinVersion: TLS 1.2` on `--tls-port`; `--client-ca` additionally loads a trust bundle and sets `RequireAndVerifyClientCert`, so an untrusted client is dropped at handshake. `validate()` rejects `--cert` without `--key`, `--key` without `--cert`, and `--client-ca` without both. The plaintext listener is dropped unless `--accept-http` is passed; the two bind separate ports so clients migrate independently. -3. **Cluster-gated**: `--cluster-dir` takes precedence over `--cert`/`--key` (which are logged as ignored) and binds a *single* listener on `--port` — deliberately not `--tls-port`, to avoid colliding with `nvpair-errors`. `nvpair-shared/splitlisten` fans connections by first byte: `0x16` (a TLS handshake record) goes to a `tls.NewListener` wrapping `Mesh.ServerTLSConfig()`, anything else to a plain sub-listener, with the peeked byte restored and a 5 s peek timeout. Both feed the same mux. +3. **Public HTTP and cluster mTLS (broker topology)**: `--cluster-dir` selects one split listener on `--port`, avoiding a second advertised port and the `--tls-port` collision with `nvpair-errors`. TLS connections go to `Mesh.ServerTLSConfig()` and the protected inventory handler; plain connections go to the independent public handler. Operator certificates do not replace the cluster pins. -Under (3) the gate is per request: `nodeInfoHandler` calls `mesh.Refresh()`, and while `mesh.Clustered()` the caller must satisfy `VerifyClientPin` — an authenticated client cert whose URI-SAN/CN UUID resolves to a byte-for-byte matching pinned peer, or this node's own leaf via self-trust — else `403`. A plaintext caller on the shared port has no client cert and is refused the same way, so joining a cluster closes the plaintext inventory in place. The server leaf is resolved *per handshake*, so an unclustered node presents none and the handshake fails with `ErrNotClustered`; because that decision precedes any handler, a `Mesh.Watch(ctx, nil)` goroutine re-derives membership every `clustertrust.RefreshInterval` (2 s) independently of traffic. +Under (3), protected requests refresh membership and require an authenticated client whose certificate matches its current exact pin, or the existing shared self-trust rule. The handler rechecks authorization after collecting the response, including on reused TLS connections, and returns `503` until owned metadata matches its current identity and contains a name, addresses, and a valid `ni` port. Departure disables this protected surface even when the keypair remains. The server leaf is chosen per handshake; the two-second mesh watcher follows changes without depending on an HTTP request. The separate public handler remains available before joining, while clustered, and after leaving. Local service measurements use owned IPC rather than the self-trust network path. -Every listener sets `ReadHeaderTimeout: 5 * time.Second` and the same `ConnState` hook (§7.6). The cluster-gated TLS sub-listener also sets `IdleTimeout` to `clustertrust.PeerListenerIdleTimeout` (105 s), longer than the calling pool's 90 s idle timeout so a peer's client always reaps a pooled connection before this side closes it. +Every listener sets `ReadHeaderTimeout: 5 * time.Second` and the same `ConnState` hook (§7.6). Both cluster-mode sub-listeners use `clustertrust.PeerListenerIdleTimeout` (105 s), longer than the calling pool's 90 s idle timeout so the client reaps its idle connection first. ### 7.6 Peer-observed addresses A multi-homed host cannot tell from its own side which of its addresses a given peer can reach — a direct-connect link works only from the machine on its far end. A peer completing a request is the one fact that settles it, and every peer's inventory poll already arrives here, so the evidence is free and continuous. @@ -231,7 +239,7 @@ A multi-homed host cannot tell from its own side which of its addresses a given - Every listener's `http.Server.ConnState` hook records the connection's local IP when the connection reaches `StateActive`, meaning a request has actually been read. That is what makes it evidence rather than a guess: a bare TCP connection could be a port scan. Loopback peers, and loopback or unspecified local addresses, are ignored, because this machine talking to itself says nothing about what others can reach. - Each address stays current for `observationTTL` (5 min) after it was last seen, which keeps a quiet peer from retracting its proof while letting an address the host stops answering on drop out within minutes. - `reportLoop` sends the full current set every `observedReportEvery` (30 s) as `nodeinfo:observed-addresses`, sorted so an unchanged set reports identically. It reports unconditionally — including an empty list — so a restarted parent relearns the set on the next tick and an expired observation is actually withdrawn. -- The broker relays each report to the scanner daemon as `discovery:set-observed-addresses`, off node-info's reader goroutine so the relay cannot stall this process's stdout. The scanner treats the set as peer-proven evidence when ranking the addresses it publishes for this node. +- The broker relays each report to scanner as `discovery:set-observed-addresses`, which ranks the host's own reachable addresses. Public requests also contribute; this evidence authenticates neither the requester nor any remotely advertised address. ### 7.7 Versioning - The route is version-prefixed (`/v1/node-info`); a breaking shape change takes a new prefix. @@ -239,21 +247,21 @@ A multi-homed host cannot tell from its own side which of its addresses a given - The binary stamps `main.Version` from `services/versions.json` at build time and prints it with `--version`. ## 8. Dependencies -- **Upstream**: `nvpair-ui-broker`, which spawns the process, passes `--log-level` and `--node-id`, owns the stdin pipe, pushes this node's cluster principal over it, reads its `nodeinfo:observed-addresses` reports from stdout, and registers the `ni` service port with the discovery daemon. -- **Downstream**: none — the service calls no other nvpair service. Its HTTP consumers are `nvpair-node-scanner` (`fetchNodeInfo`), `nvpair-manual-nodes` (`probeNodeInfo`), the Electron desktop's node-info poller, and `nvpair-tui`'s node detail screen (`pollTelemetryCmd`); `nvpair-job-scheduler` receives its GPU sample indirectly through the first two, and the scanner receives its observed addresses through the broker. +- **Upstream**: `nvpair-ui-broker` owns the process and stdio, supplies node identity and the cluster directory, pushes owned registry metadata, reads local snapshots and address observations, and registers the `ni` service port with scanner. +- **Downstream**: remote scanner and manual-node workers use protected mTLS inventory; desktop and TUI use public HTTP display inventory; the local scanner receives snapshots through the broker. Scheduler receives accepted GPU samples indirectly. This worker makes no outbound service calls. - **External**: `github.com/jaypipes/ghw` (CPU, non-macOS memory, fallback GPU); `github.com/shirou/gopsutil/v4` (macOS CPU times and memory, purego rather than cgo); `howett.net/plist` (IORegistry decoding); `golang.org/x/sys/windows` (DXGI, PDH, kernel32, registry); the `nvidia-smi` binary on Linux when present and the stock `/usr/sbin/ioreg` on macOS; first-party `nvpair-shared/applog`, `nvpair-shared/nodeid`, `nvpair-shared/noderec`, `nvpair-shared/clustertrust`, and `nvpair-shared/splitlisten`. No third-party runtime services, no network egress. ## 9. Data Ownership -- **Owned**: the static hardware readout captured at startup, the current `statsSnapshot` with its last-usable GPU sample time, and the set of peer-observed local addresses — all in-memory and transient. Nothing is retained across restarts. -- **Source of truth**: yes, for this node's hardware inventory and live utilization. Consumers cache the last successful response (the scanner keeps `lastInfo` keyed by `hostUuid` and reuses it on a transient fetch failure rather than blanking a node card), but this service is where the numbers originate. It is *not* the source of truth for node identity or cluster membership — `nvpair-cluster-manager` owns the cluster principal, and under the broker both the host UUID and that principal are handed to this process rather than derived here. +- **Owned**: static hardware, current measurements and their sample time, observed local addresses, and the latest broker-supplied registry metadata, all in memory. Broker resends metadata after restart and retains local sample age when replaying accepted snapshots. +- **Source of truth**: this host's hardware inventory and utilization originate here. Cluster-manager owns identity, membership, and pins; scanner's local registry owns the protected name, addresses, and service ports. Under the broker, `--node-id` supplies host identity and the read-only live mesh supplies membership. - **Storage**: no database, no telemetry file. The only possible write is `nodeid.Resolve` minting `/node-id.json` (`0600`, parent `0700`) when it runs standalone with no existing identity; under the broker `--node-id` short-circuits that path. `` is `%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `$XDG_CONFIG_HOME`/`~/.config/Nvidia Corporation/Personal AI Router` on Linux, and `~/Library/Application Support/Nvidia Corporation/Personal AI Router` on macOS. The cluster dir is read-only to this service. ## 10. Design Constraints - **Performance**: request handling is one atomic load plus a marshal of a few hundred bytes, and the connection hook adds one short mutex-guarded map write per request from a remote peer; all hardware work happens on the collector goroutines, and macOS keeps its once-a-second `ioreg` child process off the path that publishes CPU and memory. The 1 s tick matches the natural interval of the Windows rate counters and keeps the sample fresh for a 2 s poll without being wastefully tight. - **Scalability**: one process per node, one route, no fan-out. Load scales with the number of peers polling — at ~dozen-node scale that is a handful of requests per second against a cached body. - **Reliability**: every detection path degrades to omission instead of failure; the collector is non-nil even when PDH initialization fails entirely, and memory-used keeps publishing. A listener bind or serve failure is fatal and relies on the broker's supervisor for restart. -- **Security**: no elevation, no key material generated or held beyond an operator-supplied cert/key or the cluster leaf read from disk. TLS is pinned to 1.2 minimum in both the bring-your-own and the cluster path. The cluster path requires a client cert and enforces a byte-for-byte DER pin match, answering `403` rather than an opaque handshake failure. In the shipped broker topology the route is plaintext on the local subnet by design; the Windows installer scopes its inbound rule to `remoteip=localsubnet` so the port is never open off-link. -- **Compliance**: the payload carries GPU marketing names, a CPU model string, core count, memory and VRAM byte counts, utilization percentages, `hostUuid`, and `clusterUuid`. It carries no hostname, no hardware serial number, no MAC or IP address, no username or path, no GPU UUID, adapter LUID, or IORegistry entry ID (`statsKey` is `json:"-"`), no model names, and no inference content. `hostUuid` is a random v4 UUID (or the cluster principal), not derived from hardware, so it is a per-install pseudonym rather than a hardware identifier — but it is stable and correlatable across sessions, and `clusterUuid` additionally reveals that a set of machines belongs to one cluster, which is exactly why the response must stay on the local subnet. +- **Security**: no elevation or generated key material. Public HTTP is a supported display interface. Service measurements use owned local IPC or exact-pin remote mTLS with TLS 1.2 minimum; operator CAs cannot substitute for cluster pins. Live membership and pin checks revoke reused connections and requests still collecting a response. The Windows installer scopes the node-info inbound rule to the local subnet. +- **Compliance**: public inventory contains hardware marketing names, capacities, utilization, and identity claims. Protected inventory additionally includes the registered host name, addresses, and service ports. Neither contains hardware serial numbers, MAC addresses, usernames, paths, GPU UUIDs, adapter LUIDs, IORegistry entry IDs, model names, or inference content. Host identity is stable and correlatable across sessions; a cluster principal identifies the node rather than a shared cluster-wide identifier. ## 11. Assumptions - The parent owns the stdio pipe and is the only control peer; stdin EOF means shut down, with no reconnect and no buffering. @@ -262,7 +270,7 @@ A multi-homed host cannot tell from its own side which of its addresses a given - Windows GPU counters require Windows 10 1709 or newer; the processor counter exists on every supported version. - `nvidia-smi`, when present, is on `PATH` and returns within 3 s; a wedged driver is bounded by the timeout, not waited out. - `/usr/sbin/ioreg` exists on every supported macOS release, runs unprivileged, and returns within 3 s; its `IOAccelerator` `PerformanceStatistics` keys are undocumented and may be renamed by a macOS release. -- The parent reports this node's cluster principal; until it does, membership is unknown rather than empty, and a consumer is expected to act only on an explicit report. +- With a cluster directory, live membership supplies the cluster principal. Standalone without that directory, the parent's identity push is unknown until first supplied. Public claims never override established identity or trust. - Consumers treat an absent field as unknown and never as zero, ignore `msSince` while `telemetryValid` is `false`, and tolerate a response that carries only `{"GPUs":[],"telemetryValid":false,"msSince":0}`. - Discovery is somebody else's job: the record advertising this port is published by the node-scanner daemon, and this process never touches mDNS. @@ -274,9 +282,10 @@ A multi-homed host cannot tell from its own side which of its addresses a given - **`/proc` read or parse failure on Linux**: CPU percent or memory-used unknown. → An invalid sample, a counter reset, or zero elapsed jiffies all yield `0`, which `omitempty` drops; the next tick recovers. - **`ioreg` missing, wedged, or reporting renamed keys on macOS**: no GPU inventory or GPU stats. → The 3 s timeout bounds a wedged read and shutdown cancels one in flight, the failure warns once and every following tick retries, a renamed `PerformanceStatistics` key drops only its own metric, and CPU and memory keep publishing from the other goroutine. - **RDP phantom or virtual display adapters**: the UI would show GPUs the node cannot compute on. → Software-flag skip, name denylist, and the DirectX-registry LUID gate; when the registry is unreadable the gate is skipped with a warn rather than dropping real adapters. -- **Unpinned peer reads a cluster-gated node**: inventory disclosure to a non-member. → `VerifyClientPin` fails and the handler answers `403`; the membership answer is re-read per request and by a 2 s watch, so a peer paired after startup is admitted without a restart. -- **Malformed or undelivered cluster-identity push**: peers cannot learn this node's membership over HTTP. → A malformed payload is dropped with a warn so a wrong principal is never latched, and the parent re-pushes on every membership or pin-set change; until a push lands the field stays absent, which consumers read as "unknown" and answer by leaving their existing annotation alone. -- **Address reuse / wrong host answers a poll**: telemetry attributed to the wrong node. → The response carries `hostUuid`; the desktop skips the merge when it disagrees with the node being polled, `nvpair-tui` discards the reading and tries the node's next address, and `nvpair-manual-nodes` re-keys the entry to the reported identity. +- **Unpinned peer reads protected inventory**: the TLS handler returns `403` unless the client has a current matching pin. Public HTTP hardware remains available and omits registry metadata. +- **Owned metadata is absent or inconsistent**: protected inventory returns `503` until broker metadata agrees with the current host and cluster principal. Public display and private local snapshots remain available. +- **Malformed cluster-identity push in standalone mode**: the payload is dropped with a warning. An absent field stays unknown; public responses do not authorize service membership changes. +- **Address reuse or wrong host answers a poll**: remote service readers reject certificate or response-identity mismatches and never retry public HTTP. Frontend public reads remain hardware display data, and a verified row requires matching response identity. An unverified preview cannot promote a claim. - **Port already bound**: the process cannot serve. → `log.Fatalf` with the port in the message; the broker's supervisor restarts it and the failure is visible in the shared stderr stream. ## 13. Observability @@ -285,13 +294,15 @@ A multi-homed host cannot tell from its own side which of its addresses a given - **Alerts**: none raised by this service. It does not participate in the `errors:*` pipeline. A dead process surfaces as the broker's supervisor restart log and, downstream, as a node whose card falls back to its last cached enrichment. ## 14. Sample usage -The broker starts and spawns `nvpair-node-info --log-level info --node-id 6b1f5c2a-…`, deliberately without a cluster dir. The process resolves `hostUuid` from the flag, detects hardware — say one RTX 4090 via `nvidia-smi` with a 24564 MiB total, a 12-core Ryzen and 32 GiB via `ghw` — starts the one-second collector, binds `:14318`, and logs `serving /v1/node-info on port 14318`. The broker pushes `nodeinfo:set-cluster-identity` with this node's principal — empty here, because the node belongs to no cluster — so the process can report membership it holds no trust store to read. The broker then registers `{Service: "ni", Port: 14318}` with the discovery daemon, which folds `ni=14318` into this node's single `_nvpair-node` TXT record alongside `uuid=`, `ip=`, and the other service ports. +The broker starts `nvpair-node-info --log-level info --node-id 6b1f5c2a-… --cluster-dir `. The worker resolves the supplied host identity, detects hardware, starts the one-second collector, and opens the shared public HTTP / protected mTLS port at `14318`. Broker registers `ni=14318` with scanner and supplies the complete local registry snapshot as `nodeinfo:set-peer-metadata`. One second later the collector's first real tick lands: `/proc/stat` deltas to 7 % busy, `/proc/meminfo` reports 12 GiB used, and `nvidia-smi --query-gpu=uuid,utilization.gpu,memory.used` reports the 4090 at 37 % with 8 GiB resident. Because a utilization value parsed, the pass is usable and stamps `GPUSampledAt`; from now on responses carry `telemetryValid: true` and an `msSince` of at most about a second. The snapshot is published atomically. -A peer's node-scanner daemon sees the record, reads `ni=14318` from TXT (never the SRV port, which is non-authoritative), and issues `GET http://192.168.1.10:14318/v1/node-info` over plain HTTP — plain by static policy, because `noderec.ServiceNodeInfo.Transport()` is `TransportPlain` even against a clustered node. The handler refreshes an empty mesh (a no-op with no cluster dir), merges the static records with the snapshot by GPU UUID, and returns the document shown in §6. The daemon decodes `GPUs` / `cpu` / `memory` into the peer's `DirectoryNode`, caches it under the node's `hostUuid`, and publishes a `discovery:node-updated` event that the broker relays. It also reduces the response to the 4090's 37 % plus `telemetryValid` and `msSince` aged by its own fetch time, and hands that to its broker for the scheduler. On this host, that request's arrival recorded `192.168.1.10` as a peer-observed address, and the next 30 s report carries it to this node's own scanner. +The worker publishes the current hardware response privately at startup and every two seconds. Broker forwards it as `discovery:local-node-info` to the local scanner, including elapsed cache time. These local measurements work before joining and after leaving a cluster. -Meanwhile the Electron desktop, having learned the same node from `discovery:get-nodes`, polls that identical URL every 2000 ms with a 1500 ms timeout. Each response is checked against the polled node's id via `hostUuid`; a match with unchanged telemetry emits a `metrics:update` push, and a change upserts the node so the GPU chart and node card re-render. On shutdown the broker closes stdin, `applog.StdinRPC` observes EOF and cancels the context, the collector's ticker stops, and both servers unwind under a 3 s graceful shutdown. +An approved remote scanner treats mDNS addresses and `ni` port as connection candidates and sends `GET https://192.168.1.10:14318/v1/node-info` over pinned mTLS. The handler checks current client authorization, gathers hardware, checks authorization again, and adds matching owned metadata. The reader verifies the actual server certificate and both response identities before promoting the peer or caching its measurements. Failed authentication cannot replace an established peer or trigger an HTTP fallback. + +Desktop and TUI independently use public `http://192.168.1.10:14318/v1/node-info` for hardware display, including while clustered. Those responses omit `peer` and cannot modify service identity, trust, routing, or scheduling. On shutdown, stdin EOF cancels the worker, stops sampling, and unwinds both servers under the graceful-shutdown timeout. ## 15. Process model, CLI, and build wiring @@ -299,18 +310,18 @@ Meanwhile the Electron desktop, having learned the same node from `discovery:get | Flag | Type | Default | Effect | |------|------|---------|--------| -| `--port` | int | `14318` | HTTP port to listen on; also the port the cluster-gated listener owns | +| `--port` | int | `14318` | Public HTTP port; with a cluster directory it also serves pinned mTLS through the split listener | | `--tls-port` | int | `14319` | HTTPS port, used only with `--cert` and `--key` | | `--cert` | string | _(none)_ | TLS server certificate (PEM); requires `--key` | | `--key` | string | _(none)_ | TLS server private key (PEM); requires `--cert` | | `--client-ca` | string | _(none)_ | PEM bundle of client-cert CAs; enables `RequireAndVerifyClientCert`; requires `--cert` and `--key` | -| `--accept-http` | bool | `false` | Keep the plaintext listener alive alongside HTTPS; inert when no cert is configured | -| `--cluster-dir` | string | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`); binds one cluster-gated listener on `--port` and overrides `--cert`/`--key` | +| `--accept-http` | bool | `false` | Keep public HTTP alongside operator HTTPS; cluster mode always keeps public HTTP available | +| `--cluster-dir` | string | _(none)_ | Read-only cluster state (`node.crt`/`node.key`, admission, and pins); serves independent public HTTP and protected exact-pin mTLS handlers on the primary port | | `--node-id` | string | _(none)_ | Stable per-host UUID to report; empty resolves the local identity store | | `--version` | bool | `false` | Print `main.Version` and exit `0` | | `--log-level` | string | `""` | Registered by `applog.RegisterFlag`; precedence is flag > `NVPAIR_LOG_LEVEL` > `info` | -**Process model**: `nvpair-ui-broker` spawns it as a supervised child (`startNodeInfo`), passing `--log-level` and `--node-id`, hiding the console window on Windows, plumbing the child's stderr to the broker's own, and reading its stdout for `nodeinfo:observed-addresses` notifications. Any other notification is logged at debug and ignored, a frame with no method (such as a `log/set-level` response) is dropped, and an oversized frame is skipped with a warning. The broker writes `log/set-level` and `nodeinfo:set-cluster-identity` notifications to stdin. It shuts down on stdin EOF (the broker's `Stop` closes the pipe), `SIGINT`, or `SIGTERM`; teardown cancels the context, stops the collector, calls `Shutdown` on whichever servers exist under a 3 s timeout, and closes the splitter so both sub-listeners unwind. Running the binary standalone is supported and is what the bring-your-own TLS and `--cluster-dir` paths exist for. +**Process model**: broker supervises node-info, supplies `--log-level`, `--node-id`, and the cluster directory, owns stdio, and inherits stderr. It reads local inventory and address notifications and writes registry metadata and control frames. Restart rejects retired callbacks, retains the original receipt time of accepted local samples, and resends current metadata. The child exits on stdin EOF, `SIGINT`, or `SIGTERM`, stops the collector, gracefully shuts down listeners, and closes the splitter. Standalone public HTTP and operator HTTPS remain supported deployment modes. **Build-tag matrix**: @@ -327,7 +338,7 @@ Meanwhile the Electron desktop, having learned the same node from `discovery:get | `stats_linux.go` | `linux` | `/proc` + `nvidia-smi` collector | | `stats_darwin.go` | `darwin` | gopsutil + `ioreg` collector | | `stats_other.go` | `!windows && !linux && !darwin` | no-op collector, `luidKey` stub | -| `stats_test.go`, `observed_test.go`, `observed_wiring_test.go`, `identity_test.go`, `cluster_mtls_test.go`, `cluster_identity_test.go`, `ioreg_parse_test.go` | _(none)_ | merge/omitempty and telemetry validity, the observed-address recorder and its listener wiring, identity resolution, the mTLS gate, the cluster-principal wire states, and IORegistry parsing — run on every platform | +| Matching `*_test.go` files beside their primary implementation | _(none)_ | hardware response fields, sample validity and age, observed addresses, identity resolution, public HTTP and protected mTLS behavior, owned IPC, and IORegistry parsing | | `stats_linux_test.go` | `linux` | `/proc` and `nvidia-smi` parsers | | `gpu_darwin_test.go`, `stats_darwin_test.go`, `memory_detect_darwin_test.go` | `darwin` | IORegistry GPU assembly and its timeout, the two-goroutine collector, and physical-memory detection | | `stats_windows_test.go`, `gpu_windows_test.go` | `windows` | LUID formatting and Win32 struct-size assertions | diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index ae2376e09..e7ce04c9a 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -71,6 +71,13 @@ nothing to answer and the caller must not wait: Emitted as the directory changes. Each carries the affected `DirectoryNode` (identity, advertised service ports, enrichment, `trusted`). +Records also carry `discoveryRecordId` and `identityVerification`. The local registry +record is `local`; browsed records are `unverified`, including when their claimed +principal has a pin. `pinned` denotes authenticated identity in the shared +contract. Discovery record IDs identify connection candidates independently of UUID +and principal claims. Their address order does not affect the ID. The existing +`trusted` pin annotation alone does not prove endpoint ownership. + - `discovery:node-discovered` — a new node entered the directory - `discovery:node-updated` — a known node's record or enrichment changed. A re-observation that changes nothing emits no event, so `lastSeen` is a last-*change* stamp rather than a liveness heartbeat - `discovery:node-removed` — a node aged out (identity + last-known fields) diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 42226db3d..778ebaded 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -876,14 +876,16 @@ func (d *daemon) publishSelfLocked() { svcs[k] = noderec.ServiceStatus{Port: port} } node := noderec.DirectoryNode{ - HostUUID: rec.HostUUID, - Name: d.instance, - IP: ip, - IPs: ips, - ClusterUUID: rec.ClusterUUID, - Trusted: trusted, - Services: svcs, - LastSeen: time.Now().Unix(), + DiscoveryRecordID: "local:" + rec.HostUUID, + IdentityVerification: noderec.IdentityLocal, + HostUUID: rec.HostUUID, + Name: d.instance, + IP: ip, + IPs: ips, + ClusterUUID: rec.ClusterUUID, + Trusted: trusted, + Services: svcs, + LastSeen: time.Now().Unix(), } // Enrich over loopback, never the advertised LAN ip=: a first-run Windows // firewall block on inbound to our own LAN address must not blank the local diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index cfc2e8228..51f85480e 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -333,3 +333,12 @@ func TestReachableSelf_DialsLoopback(t *testing.T) { } assert.False(t, d.reachable(peer), "a same-hostname peer advertising an unroutable LAN ip must not inherit the self loopback dial") } + +func TestPublishSelf_LocalRecordHasLocalVerificationAndDiscoveryRecordID(t *testing.T) { + d := newSelfTestDaemon("self-uuid", "192.0.2.10") + d.publishSelf() + node, ok := d.dir.get("self-uuid") + require.True(t, ok, "local registry record must exist") + assert.Equal(t, noderec.IdentityLocal, node.IdentityVerification) + assert.Equal(t, "local:self-uuid", node.DiscoveryRecordID) +} diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index 70b7d04f3..aafe792cb 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -4,8 +4,11 @@ package main import ( + "crypto/sha256" + "fmt" "log/slog" "sort" + "strconv" "sync" "time" @@ -50,19 +53,32 @@ func toDirectoryNode(raw RawNode, trusted bool) (noderec.DirectoryNode, bool) { svcs[k] = noderec.ServiceStatus{Port: port} } return noderec.DirectoryNode{ - HostUUID: rec.HostUUID, - Name: raw.ID, - IP: ip, - IPs: ips, - ClusterUUID: rec.ClusterUUID, - Trusted: trusted, - Services: svcs, + DiscoveryRecordID: nodeDiscoveryRecordID(raw), + IdentityVerification: noderec.IdentityUnverified, + HostUUID: rec.HostUUID, + Name: raw.ID, + IP: ip, + IPs: ips, + ClusterUUID: rec.ClusterUUID, + Trusted: trusted, + Services: svcs, // Models is enriched separately from engine-manager's em /v1/models // (daemon.enrichModelsAt), not carried in the record's TXT. LastSeen: time.Now().Unix(), }, true } +// nodeDiscoveryRecordID identifies a connection candidate, not its claimed owner. +// Address ordering and UUID/principal claims do not change that identity. +func nodeDiscoveryRecordID(raw RawNode) string { + rec := noderec.ParseTXT(raw.TXT) + addresses := append([]string(nil), netpick.Candidates(raw.TXT, raw.Addresses)...) + sort.Strings(addresses) + fields := []string{raw.ID, raw.Host, strconv.Itoa(rec.Services[noderec.ServiceNodeInfo])} + fields = append(fields, addresses...) + return fmt.Sprintf("observation:%x", sha256.Sum256([]byte(fmt.Sprintf("%q", fields)))) +} + // directory is the daemon's view of all LAN nodes (this node included), keyed by // hostUuid. Mutations come from the browse-event goroutine; reads come from // discovery:get-nodes handlers, so it's an RWMutex map. diff --git a/services/nvpair-node-scanner/directory_test.go b/services/nvpair-node-scanner/directory_test.go index f2474038f..92276e1fe 100644 --- a/services/nvpair-node-scanner/directory_test.go +++ b/services/nvpair-node-scanner/directory_test.go @@ -140,3 +140,70 @@ func TestDirectory_UpdatesRemovesAndSnapshotsNodes(t *testing.T) { require.Len(t, all, 1, "after remove, snapshot") require.Equal(t, "b", all[0].HostUUID, "after remove, snapshot (%v)", all) } + +func observationFixture() RawNode { + return RawNode{ + ID: "workstation", + Host: "workstation.local.", + Port: noderec.SRVPort, + Addresses: []string{"192.0.2.20", "192.0.2.21"}, + TXT: []string{"v=1", "uuid=claimed-host", "cluster-uuid=claimed-principal", "ni=14318"}, + } +} + +func TestToDirectoryNode_AddressOrderAndClaimChangesPreserveDiscoveryRecordID(t *testing.T) { + test := func(name string, candidate RawNode) { + t.Run(name, func(t *testing.T) { + original, ok := toDirectoryNode(observationFixture(), false) + require.True(t, ok, "original observation must be projected") + next, ok := toDirectoryNode(candidate, false) + require.True(t, ok, "candidate observation must be projected") + require.NotEmpty(t, original.DiscoveryRecordID) + assert.Equal(t, original.DiscoveryRecordID, next.DiscoveryRecordID) + }) + } + + test("repeated observation", observationFixture()) + reordered := observationFixture() + reordered.Addresses = []string{"192.0.2.21", "192.0.2.20"} + test("resolved address order changes", reordered) + claims := observationFixture() + claims.TXT = []string{"ni=14318", "uuid=another-claim", "v=1", "cluster-uuid=another-principal", "ol=11434"} + test("identity and other service claims change", claims) +} + +func TestToDirectoryNode_DifferencesInDiscoveryFieldsYieldDifferentDiscoveryRecordIDs(t *testing.T) { + test := func(name string, candidate RawNode) { + t.Run(name, func(t *testing.T) { + original, ok := toDirectoryNode(observationFixture(), false) + require.True(t, ok, "original observation must be projected") + next, ok := toDirectoryNode(candidate, false) + require.True(t, ok, "candidate observation must be projected") + require.NotEmpty(t, original.DiscoveryRecordID) + require.NotEmpty(t, next.DiscoveryRecordID) + assert.NotEqual(t, original.DiscoveryRecordID, next.DiscoveryRecordID) + }) + } + + instance := observationFixture() + instance.ID = "another-instance" + test("different service instance", instance) + host := observationFixture() + host.Host = "another-host.local." + test("different resolved host", host) + address := observationFixture() + address.Addresses = []string{"192.0.2.30"} + test("copied UUID at another resolved address", address) + advertised := observationFixture() + advertised.TXT = append(advertised.TXT, "ip=192.0.2.30") + test("different advertised connection address", advertised) + port := observationFixture() + port.TXT = []string{"v=1", "uuid=claimed-host", "cluster-uuid=claimed-principal", "ni=14328"} + test("different node-info port", port) +} + +func TestToDirectoryNode_TrustedFlagDoesNotVerifyIdentity(t *testing.T) { + node, ok := toDirectoryNode(observationFixture(), true) + require.True(t, ok, "observation must be projected") + assert.Equal(t, noderec.IdentityUnverified, node.IdentityVerification) +} diff --git a/services/nvpair-node-scanner/spec.md b/services/nvpair-node-scanner/spec.md index 3a78cd3e4..3c7757567 100644 --- a/services/nvpair-node-scanner/spec.md +++ b/services/nvpair-node-scanner/spec.md @@ -3,423 +3,563 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Microservice: Node Scanner (`nvpair-node-scanner`) +# Microservice: Node Scanner `nvpair-node-scanner` ## 1. Purpose -Owns node discovery for this host. It is the single mDNS advertiser and the single mDNS browser on the machine: no other worker runs a responder or a resolver. Every local service that wants to be reachable from the LAN hands this daemon a short service key and a port (`discovery:register`); the daemon folds them into **one** `_nvpair-node._tcp` record carrying this node's identity, its LAN address, and one `key=port` TXT entry per registered service. Simultaneously it browses `_nvpair-node._tcp`, reconciles the answers into a directory of every node on the subnet (this node included), decorates each entry with the data that does not fit in a DNS record — hardware inventory from `nvpair-node-info` and the model inventory from `nvpair-engine-manager` — and emits `discovery:node-discovered` / `discovery:node-updated` / `discovery:node-removed` deltas. Separately it samples every node's GPU utilization every couple of seconds and reports it as compact `discovery:node-telemetry` for the scheduler. - -It exists so that discovery has exactly one implementation per host. One record makes this node's advertised surface atomic — a peer either sees it with all of its ports or not at all — and one browser makes "who is on the network, at what address" a single answer every consumer shares rather than a per-worker approximation. It also confines the hard parts of LAN discovery to one place: the per-interface multicast send path a multi-homed host depends on to be visible at all, the choice of which of this host's own addresses to publish, and the eviction policy that keeps a healthy node — including one too saturated by its own inference load to answer — from being dropped. `nvpair-ui-broker` relays the resulting directory to every other worker and to the desktop UI, so no other process needs a responder, a resolver, or an opinion about addressing. +The Node Scanner (Scanner for short) discovers computers on the local network and maintains a directory of their +identities, services, hardware, and models. It announces this computer, the host, +as one node, even when several services run on it. The broker distributes node +records and verified telemetry (hardware measurements) to services such as +proxies and scheduler. Desktop and the text interface (TUI) use those node +records and may read public hardware measurements separately for display. ## 2. Scope **In scope** -- Advertise exactly one `_nvpair-node._tcp` mDNS record per host, built from the registered service set plus the daemon-stamped identity (`uuid=`, `cluster-uuid=`, `ip=`, `ips=`). -- Rank this host's own publishable addresses from evidence it observes — peer-proven addresses, interfaces facing a discovered peer, interfaces whose sends keep failing — and publish the best as `ip=` and up to four as `ips=`. -- Accept `discovery:register` / `discovery:unregister` / `discovery:update-txt` and re-advertise only on a real change to the record. -- Browse `_nvpair-node._tcp`, reconcile scans into a directory keyed by host UUID, and emit per-node deltas. -- Decorate directory entries with `/v1/node-info` inventory (GPU, CPU, memory) and `/v1/models` inventory (flat, per-engine, and per-engine loaded sets), dialing each node at every address it published until one answers as that node. -- Sample every node's `/v1/node-info` on a two-second cadence and emit compact GPU telemetry (`discovery:node-telemetry`) for the broker's scheduler feed, without touching the directory. -- Converge peer cluster membership and model inventories on one periodic sweep that is independent of mDNS change events. -- Re-resolve and re-advertise this node's identity and LAN address on demand (`discovery:reload-identity`, `discovery:reload-trust`) and on a network change. -- Annotate a browsed peer as `trusted` when this node holds a cluster pin for the principal that peer advertises, and re-derive that annotation across the whole directory on `discovery:reload-trust`. -- Evict a node only after a minute of mDNS silence during which every liveness probe also failed; keep it while it answers an `ni` / `em` probe as itself (confirmed over `/v1/node-info`), has a fresh enrichment, or has recently served inference. -- Treat a scan that loses every known node at once as this host's own failure, for a bounded number of scans. -- Drop a directory record proven superseded at its address, so a machine that comes back under a fresh identity is not carried twice. + +- Using multicast DNS (mDNS), a protocol for local-network discovery, to announce + a maintained list of local services (`_nvpair-node._tcp`). +- Discovery, address selection, a node directory that clients can query, and + notifications when records change. +- Public hardware details over HTTP for frontend display, and authenticated + details for services using approved peer nodes in the cluster. +- GPU, CPU, memory, and model enrichment: gathering extra details to add to a + discovered node's directory record, including models loaded by each engine. + These details are sent to broker clients, not broadcast in mDNS announcements. +- Regular telemetry reads, sample-age tracking, retries, and anti-flap behavior: + delaying removal to avoid treating short discovery gaps as disconnects. +- Identity and trust updates while running, using saved peer certificates + (certificate pins) to check that each peer presents the expected certificate. **Out of scope** -- Serving the directory to clients or fanning it out to workers — `nvpair-ui-broker` owns `discovery:subscribe`, the relay directory, and the client-facing `discovery:get-nodes` / `discovery:nodes-changed` snapshots (§7.7). -- Cluster identity, PIN pairing, certificate issuance, pin management, and membership — `nvpair-cluster-manager`. This daemon only *reads* the cluster directory through `nvpair-shared/clustertrust`. -- Producing the hardware inventory it fetches — `nvpair-node-info`. -- Producing the model inventory it fetches, and every model or engine lifecycle operation — `nvpair-engine-manager`. -- Manual (user-entered) nodes — `nvpair-manual-nodes` probes them and `nvpair-ui-broker` merges them into its own store (§7.7). -- Routing and target selection — `nvpair-proxy` and `nvpair-job-scheduler`. This daemon reports raw GPU utilization; turning it into scheduling pressure is the scheduler's job. -- Persisting anything. The only file this daemon can create is the fallback node id, and only indirectly through `nvpair-shared/nodeid` (§9). + +- Pairing, certificate generation, membership decisions, and saving pins, + owned by `nvpair-cluster-manager`. +- Hardware collection, owned by `nvpair-node-info`, and engine startup/model + operations, owned by `nvpair-engine-manager`. +- Sending inference requests and choosing where to run them, owned by proxies + and scheduler. +- Frontend network reads or cryptography, and a separate system for signing + node details. ## 3. Key Use Cases -- **Advertise this node**: the broker relays a `discovery:register` per local service as it starts each worker — `ni` (node-info, `14318`), `wl` (workload-manager, `14320`), `er` (errors, `14319`), `em` (engine-manager, `14322`), `ec` (engine remote control, `14323`, only when a cluster dir is configured), `cl` (cluster-manager, `14321`) — plus `ol` / `lm` / `lc` from its engine pollers at the listen port of the matching `nvpair-proxy` facade, never the engine's own port. The registry merges each into `NodeRecord`, `Responder.UpdateTXT` re-announces, and `publishSelf` refreshes this node's own directory entry. -- **Discover a peer**: a scan returns a peer's `_nvpair-node` record; `toDirectoryNode` projects TXT into a `DirectoryNode` keyed by `uuid=`, enrichment fills in hardware and models, and `discovery:node-discovered` carries the whole entry to the broker. -- **A peer changes**: a peer starts an engine (a new `ol=` key) or moves address (`ip=` / `ips=`); the order-insensitive record compare in `nvpair-shared/discovery` sees a real change, re-enrichment runs, and `discovery:node-updated` fires. A scan that returns the same addresses and TXT in a different order emits nothing. -- **A peer leaves**: it stops answering. From its third consecutive missed scan (about 15 s) the daemon dials its `ni` port and then its `em` port at each published address on **every** scan, and any answer resets the miss counter. Only after twelve consecutive misses (a full minute) with every probe failing is the entry dropped, its enrichment caches forgotten, and `discovery:node-removed` emitted. A clean shutdown multicasts an RFC 6762 goodbye (TTL 0 PTR), which clears the record from mDNS caches so it is absent from the very next scan. The browser has no goodbye fast path, though: the node still takes the miss path, its probes are refused from about 15 s on, and it is evicted at the one-minute threshold. -- **A peer is saturated by its own inference load**: its control plane is starved, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. The one-minute window absorbs most such bursts, and inference response bytes seen from it in the last 60 s (`discovery:node-activity`, relayed by the broker from whichever proxy saw them) keep it with no probe at all — the only liveness evidence that gets stronger the busier the node is. -- **This host stops hearing multicast**: a process too starved to drain its multicast socket within the scan window hears silence from every node at once. When two or more nodes are known and a scan returns none of them, no node is penalized, for up to six consecutive scans (about 30 s); after that they count normally. Independent machines do not leave the network in the same five seconds. -- **Reach a multi-homed peer**: a peer publishing a LAN address and a direct-connect link address lists both in `ips=`. Enrichment, the periodic sweep, telemetry, and the liveness probe dial each in the peer's own order, skip an answer from a different `hostUuid`, and remember per node and service which address answered so later rounds ask only that one. -- **Feed the scheduler**: every ~2 s the daemon GETs each node's `/v1/node-info`, reduces it to the highest GPU utilization plus `telemetryValid` and `msSince` (advanced by the request's own latency), and emits `discovery:node-telemetry`. The broker caches it for `nvpair-job-scheduler`. The directory is not touched. -- **A peer is wiped and reinstalled**: it mints a fresh `uuid=` but keeps its LAN address, its hostname, and its fixed service ports, so a connect-only probe would reach the new incarnation and report the old record alive forever. Two paths collapse the duplicate, and both delete only on the same proof — a `/v1/node-info` answer naming a different host. If the old record ages into the miss threshold, `identityAlive` overrules the successful TCP dial; if the new record is browsed first, `supersedingUpsert` puts the same question to the arriving record's `ni` port before the new entry is stored. -- **Model inventory converges without an mDNS event**: a peer pulls or deletes a model, or starts an engine late. Nothing about its mDNS record changes, so the browse-driven path never re-runs; the 15-second sweep re-fetches every `em` node's `/v1/models`, and only a genuinely changed inventory produces one `discovery:node-updated`. -- **A peer's cluster departure is announced but not received**: the peer leaves and re-advertises without `cluster-uuid=`, but this host never receives that record, and the liveness probe keeps the frozen entry alive precisely because the peer is still reachable. The same 15-second sweep reads `clusterUuid` from the peer's own `/v1/node-info` and clears the stale principal, so the client stops suppressing an invite the peer would now accept. -- **This node joins a cluster while running**: `nvpair-cluster-manager` writes the keypair and activates the admission, and the broker relays `discovery:reload-trust`; `reconcileIdentity` sees live membership disagree with what the registry publishes, `cluster-uuid=` appears in the record, and peers can key their pins on it. No restart. -- **A peer is paired after it was already discovered**: its mDNS record does not move when this node's pin lands, so the browse path never re-asks whether it is trusted and it would stay unroutable for the life of the process. `discovery:reload-trust` re-derives `trusted` for every directory entry against the current pin set and emits one `discovery:node-updated` per entry whose answer changed. -- **Edge case — self is never browse-driven**: Windows frequently does not loop a host's own multicast back to the browsing socket, so aging this node like a peer would evict it from its own directory. `onBrowse` discards any event whose `uuid=` matches the registry's `HostUUID`; this node's entry comes only from `publishSelf`. -- **Edge case — self enrichment behind a first-run firewall prompt**: a fresh Windows install blocks inbound to this host's own LAN address, which would blank the local node's metrics and models. Self is always dialed at `127.0.0.1` while the entry still displays the advertised LAN `ip=`, so the local card is populated regardless. -- **Edge case — identity re-stamp on a fresh host**: the scanner spawns before `nvpair-cluster-manager` and mints its own `node-id.json`, so it initially advertises a UUID that is not the cluster principal. Once the broker triggers `discovery:reload-identity`, the daemon re-resolves, moves its directory entry to the new UUID (carrying the enrichment caches across), emits `discovery:node-removed` for the stale one, and republishes. -- **Edge case — a record with no `uuid=`**: it cannot be keyed by a stable identity, so it is warned about and skipped rather than keyed by the mutable, non-unique hostname. A later `removed` for it is a no-op because it was never stored. -- **Edge case — two hosts sharing a hostname**: the browser keys by `uuid=` rather than the mDNS instance name, so they stay distinct entries. A key whose `uuid=` changes underneath it logs a collision / host-replacement warning. -- **Edge case — transient enrichment failure**: a failed `/v1/node-info` or `/v1/models` fetch reuses that node's last successful values instead of blanking the entry; a node that was never enriched simply stays un-enriched. -- **Edge case — a node removed or re-addressed mid-sweep**: an in-flight `/v1/models` result is applied only when the node still exists *and* still advertises the same `em` endpoint (`ip` + port) the fetch targeted. Otherwise it is discarded, so it can neither resurrect a removed node nor overwrite a re-addressed one. -- **Edge case — a peer that reports no `clusterUuid`**: silence is not an answer. `NodeInfoResponse.ClusterUUID` is a pointer so an absent field is distinguishable from an explicit `""`, and only the explicit value moves the annotation — reading absence as "unclustered" would mark a clustered peer invitable, which is the failure the annotation exists to prevent. A failed fetch is treated the same way. -- **Edge case — a stranger answering at a peer's address**: the membership sweep compares the `hostUuid` in the response against the entry it is refreshing and skips a mismatch rather than writing one machine's membership onto another's entry. That entry ages out through the liveness probe's identity check, which owns the eviction decision. -- **Edge case — a supersede candidate that moves during the probe**: the proof is a network read taken without the directory lock, so `upsertEvicting` re-validates each verdict against the entry actually stored. A record removed, re-addressed, renamed, or refreshed in the meantime is left alone and judged again from its new state. -- **Edge case — a node advertising neither `ni` nor `em`**: the liveness probe has nothing to ask, so a threshold-missed identity-only node (or one advertising only engine keys such as `ol`, `lm`, or `lc`) evicts on the ordinary path. Inference ports are never dialed as a liveness signal. +- **Publish this host**: the broker registers local service ports, and Scanner + announces the host's identity, addresses in preferred order, and service ports + as one node. +- **Discover an unpaired node**: Scanner records its mDNS announcement as an + unverified discovery. Desktop and TUI can read public hardware details for that + row without adding those measurements to the service directory or caches. +- **Discover a cluster peer**: Scanner verifies its node-info server against the + saved certificate, then updates its directory record and fetches its model list + using mTLS (mutual TLS), an encrypted connection where both nodes present + certificates. Each checks the other's certificate against its saved pin. +- **Show live hardware usage**: desktop and TUI read public node-info for their + hardware displays. Services read authenticated peer measurements, with a + smaller GPU-usage message sent through the broker to scheduler. +- **Recover after a network change**: Scanner tries other known addresses and + refreshes inventory without depending on receiving every network announcement. +- **Follow cluster changes**: Scanner reloads identity and pins when nodes join + or leave, updating announcements and which peers can be used without a restart. ## 4. Open Questions / Risks -- **Risk — mDNS staleness, by design**: an ungracefully departed node lingers for twelve missed scans (about a minute), up to six scans longer if it vanished in a scan that lost every node, and as long as it keeps answering a probe as itself. The window is long on purpose: evicting a saturated-but-working node fails its in-flight work, while a lingering departed one costs only a failed-and-retried request. A new node still appears within the next 5-second scan. Consumers that route on the directory must tolerate a target that has just gone away. -- **Risk — multicast-hostile networks**: on a segment that blocks or does not forward multicast (guest Wi-Fi, client isolation, some VPN and virtual adapters) nothing is discovered and nothing discovers this node. There is no unicast discovery fallback in this daemon; the only path to such a node is a user-entered manual node, which `nvpair-manual-nodes` probes and the broker merges (§7.7). -- **Risk — interface flapping**: VPN and dock toggles, DHCP re-lease, and sleep/wake re-IP all change the address the node should advertise. `netmon` debounces bursts for 1500 ms and only fires on an observed change, and `reloadIdentity` is idempotent for unchanged fields, so a flap costs a re-resolve and at most one re-announce. A flap that alternates between two equally-ranked addresses will alternate the advertised `ip=`. The published order is also re-ranked on every 15-second sweep as evidence arrives, so a newly discovered peer, a newly peer-proven address, or an interface whose sends start failing can move `ip=` with no interface change at all; each such move is one re-announce. -- **Risk — the responder is built once**: `mdns.NewResponder` fails when no multicast-capable IPv4 interface with a non-loopback address exists (Wi-Fi off at boot, a laptop booted undocked). The daemon logs a warning and continues browse-only, and nothing rebuilds it when an interface later appears, so the node stays unadvertised until the worker is restarted. `watchAddress` is started inside the same responder-present branch, so on such a host `ip=` also stops tracking the interface: the self entry's address is then re-derived only when the broker sends `discovery:reload-identity` or `discovery:reload-trust`. The browser is unaffected (it builds a fresh resolver per scan). -- **Risk — TXT size limit is guarded but not enforced**: `noderec.ValidateTXTSize` implements the RFC 6763 §6.1 255-byte per-string check, but nothing in the daemon calls it. With the model list moved off the record every current key is short and bounded (`v`, `uuid`, `cluster-uuid`, `ip`, `ips` — which `NodeRecord.TXT()` itself caps at four addresses — and one `key=port` per service), so no realistic overflow exists today — but a future long key would be advertised unchecked. Wiring the check into `registry.txt()` is the obvious fix. -- **Risk — hardware inventory freshness is event-driven only**: the daemon GETs each peer's `/v1/node-info` every ~2 s for telemetry and every 15 s for membership, but the telemetry pass keeps only the highest GPU utilization for the scheduler and the membership pass only `hostUuid` and `clusterUuid`; both discard `GPUs`, `cpu`, and `memory`. The directory's inventory fields are written only by a browse change and by `publishSelf`. A stable peer's GPU / CPU / memory utilization is therefore as old as its last record change even though a fresh answer was just on the wire, and neither fetch stamps `lastInfoAt`, so neither spares the liveness probe a dial. The desktop mitigates the staleness by polling each node's `/v1/node-info` itself; a headless consumer of the directory does not get live utilization. -- **Risk — eviction rests on `hostUuid` agreement between two processes**: both delete paths act on a `/v1/node-info` answer naming a different host, which assumes node-info reports exactly the `uuid=` the scanner advertises. The broker keeps them aligned by passing its resolved node id to node-info, and `nvpair-cluster-manager` adopts an existing node id rather than minting a rival one; if that invariant is ever broken the probe reads the skew as a wipe and evicts a live peer. The daemon's own safety argument also assumes a clustered peer's node-info refuses a plaintext caller, which holds only for node-info run with `--cluster-dir` — the broker deliberately does not pass it, so in the shipped configuration the probe reaches a conclusion for clustered peers too. -- **`lastSeen` semantics (open — needs a consumer decision)**: `DirectoryNode.lastSeen` is stamped when an entry is *projected*, so it advances only when the record actually changes, not on every successful scan. It is a last-change timestamp, not a liveness timestamp. A consumer that treats it as "seconds since we last heard from this node" will read a healthy, unchanging node as increasingly stale; `supersedeCandidates` uses the `supersedeMinAge` gap only as a cheap pre-probe filter for exactly that reason, never as evidence. -- **`ServiceStatus.probe` (open — needs scope)**: the field is declared and always `""` (`ProbeUnknown`). The liveness mechanism that matters — probe-before-evict — is implemented, but per-service `reachable` / `unreachable` / `inaccessible` reporting is not, because it would add one dial per service per browse. Until it lands, consumers must not read `probe` as a health signal. -- **`discovery:get-nodes` is served but unused (open)**: the daemon implements it, and it is the natural way to resynchronize a subscriber, but the broker builds its relay directory purely from the event stream and never calls it. It is exercised only by unit tests today. -- **Future — restart replay by epoch**: `noderec.ReadyParams` carries an optional `epoch` so a parent could replay registrations when it observes a new one. The daemon does not send it (its `ready` payload is `{version}` only), and the broker instead replays its registration cache unconditionally on every scanner spawn. The mechanism is documented in the shared type but not implemented here; the current behavior is correct because the broker owns the cache. -- **Future — TLS for node-info enrichment**: the `--client-cert` / `--client-key` / `--ca-bundle` scaffolding is complete and dormant. Nothing in the product passes these flags, and the choice is deliberately global (operator flags only) rather than inferred per node from `cluster-uuid=` or a TXT hint, so the transport policy stays static. -- **Discovery is unauthenticated by nature**: appearing on `_nvpair-node` is a claim, not proof. `uuid=`, `cluster-uuid=`, `ip=`, and every advertised port are self-asserted by whoever sent the packet. `trusted` is the only trust-bearing field, and it is derived locally from the pin set `nvpair-cluster-manager` owns — never from anything on the wire. +- **Public inventory for frontend display**: public HTTP remains a supported + interface, including while clustered. It keeps desktop and TUI hardware + displays available without giving either client cluster keys. HTTP does not + verify who supplied the measurements, so those values are display data only. + Scheduling, trust, routing, and other service decisions must use authenticated + remote data or measurements received from a locally owned worker. +- **Unreachable does not mean removed**: a failed authenticated read may mean + an overloaded or unreachable peer. It does not prove that the peer left the + cluster, and an unauthenticated response cannot prove that either. +- **Service coordination**: node-info connections, shared records, broker + caches, and service consumers must be updated together. A paired node that + does not support authenticated inventory reads cannot supply verified details + until upgraded. Unauthenticated data cannot be used in their place. ## 5. Requirements **Functional** -- Accept `discovery:register`, `discovery:unregister`, `discovery:update-txt`, `discovery:get-nodes`, `discovery:reload-identity`, `discovery:reload-trust`, `discovery:set-observed-addresses`, `discovery:node-activity`, `shutdown`, and `log/set-level` from the parent as either requests (answered) or notifications. The parent sends `discovery:node-activity` as a notification. -- Ignore a `discovery:register` with an empty `service` or a zero `port`, and report no change for a re-register of an already-registered port, so an idempotent replay does not re-announce. -- Build the advertised TXT deterministically: `v`, then `uuid`, then `cluster-uuid` when set, then `ip` when set, then `ips` when there is more than one address (capped at four), then known service keys in a fixed order, then any unknown service keys sorted. -- Stamp identity centrally: `uuid=` from `nodeid.Resolve`, `cluster-uuid=` from live cluster membership, `ip=` and `ips=` from `netpick.LocalCandidates` ranked with this host's own evidence. A registering worker supplies only its key and port. -- Re-rank this host's published addresses on every peer sweep and whenever the peer-observed set changes, replacing rather than merging that set so an address peers stop reaching stops counting as proof. -- Advertise `cluster-uuid=` only while actually a cluster member, never merely because a keypair from a past membership is still on disk. -- Re-derive `trusted` for every directory entry on `discovery:reload-trust`, unconditionally rather than behind a change signal, and emit one `discovery:node-updated` per entry whose answer moved and nothing for the rest. -- Move a peer's `clusterUuid` only on its own explicit report, from a `/v1/node-info` response whose `hostUuid` matches the entry being refreshed; leave the annotation untouched on a failed fetch, an absent field, or a mismatched host. -- Delete a directory record only on proof that the machine at its address is someone else — an address, a hostname, and a `lastSeen` gap taken from an unauthenticated record may nominate, never decide. -- Key the directory by `uuid=`; skip and warn on a record without one. -- Read every service port from TXT and ignore the record's SRV port. -- Dial a node at each address it published, in its own ranked order (`netpick.Candidates`: its `ip=` / `ips=` list, then any other resolved address); accept an answer only from the node's own `hostUuid`, and remember per node and service which address answered. -- Fetch `/v1/node-info` from a node advertising `ni` over plain HTTP, and `/v1/models` from a node advertising `em`; retain the last successful values on failure. -- Sample `/v1/node-info` from every node advertising `ni` on a two-second cadence with stable per-node jitter, back off a failing remote node to 4, 8, 16, then 30 s and reset on success, never back off loopback, and emit `discovery:node-telemetry` without changing the directory. -- Evict only after twelve consecutive missed scans; from the third, probe on every scan and recover the node on any answer; skip the probe for a node with an enrichment under 10 s old or inference activity under 60 s old; and when two or more nodes are known and a scan returns none of them, count no misses for up to six consecutive scans. -- Fetch a peer's `/v1/models` only over cluster mTLS pinned to the principal it advertises; attempt nothing when no pin is held. Fetch this node's own over loopback in plaintext, in every membership state. -- Compare model inventories order-insensitively across the flat list, the per-engine map, and the per-engine loaded map, and emit at most one `discovery:node-updated` per real change. -- Emit `ready` once at startup with the build version. + +- Scanner owns the host's network announcement. Other workers register services with + Scanner through the broker instead of announcing separate discovery records. +- The public HTTP node-info endpoint remains available for frontend hardware + displays, including while clustered. Its data cannot enter scheduling caches, + verified peer records, routing inputs, trust decisions, or other service logic. +- Scanner uses mDNS identity, name, addresses, and ports to decide where to try + connecting. Finding a pin for an advertised identity selects the certificate + to check against, but does not prove who sent the announcement. +- Scanner marks a remote peer verified only after pinned mTLS and checks on the + identity in its response. It uses the name, addresses, ports, and hardware from + that authenticated response to update its record. +- The local host, verified peers, unverified discoveries, and manually configured + servers have separate records. A claimed node identifier, a UUID, cannot cause + unverified data to be merged into a local or verified peer record. +- Scanner refreshes inventory every 15 seconds and reads healthy node-info + servers every two seconds with stable per-node jitter, a small fixed timing + offset that spreads requests across nodes. Failed remote reads use backoff, + increasing the wait: 4, 8, 16, then 30 seconds between retries, returning to the + normal interval on success. Local reads stay at the normal interval. +- Scanner retains the last successfully read inventory during temporary failures. + Samples retain their validity and age, including time spent in requests, queues, + and caches. Forwarding or resending old data must not make it look new. +- Scanner sends the highest GPU utilization on each node for scheduling, using + pinned mTLS for remote samples and private interprocess communication (IPC) + with the local worker for local samples. Public HTTP node-info never supplies + scheduling data, including for manually configured nodes. +- A discovery record becomes eligible for removal after 12 consecutive missed + five-second scans. Scanner retains it when there is accepted evidence that the + node is available. If more than one node is known and a scan returns none, + Scanner ignores at most six consecutive empty scans before counting misses + normally. For a paired peer, continued availability must be shown by + authenticated inventory reads or inference responses, not just an open TCP + port or unverified network claims. + If a verified peer remains after its last discovery expires, Scanner continues + checking availability during inventory refresh. After a minute without + authenticated inventory or inference activity, Scanner removes its record and + caches while preserving membership and saved certificates. +- Scanner applies trust changes while running. Removed or replaced pins clear + verification status and scheduling samples, close affected connections, and + prevent pending responses from restoring the old state. **Non-functional** -- Never let a browse event mutate this node's own directory entry. -- Serialize every mutation of this node's own entry so a registration burst and a concurrent identity or address change cannot interleave into a torn entry. -- Run each half of the peer sweep concurrently, bounded at 16 in flight, and to completion before the next tick, so overlapping sweeps are impossible and one slow peer never consumes another's timeout budget. Bound the telemetry loop at 16 in flight as well, with at most one request per node outstanding across ticks. -- Hold at most one connection per node-info origin, and queue a caller for that origin before its request timeout starts, so a caller waiting behind a slow healthy response does not time out before its own request is sent. -- Run the pre-eviction liveness probe outside the browser lock so a dead peer cannot stall readers, and gather a supersedence proof outside the directory lock, re-validating the verdict when the write lands. -- Bound every outbound fetch: 3 s for node-info enrichment (with a 1 s connect and TLS-handshake budget), 6 s for models, 1 s per liveness dial, and 1 s for the probe's own node-info read — the probe runs inside the browser's scan cycle, so it gets a tighter budget than enrichment. -- Reach the identity read only after the TCP sweep has already found the address answering, so a departed node costs no HTTP request at all. -- Survive an unreadable or absent cluster directory, an unavailable network monitor, and a responder that could not be built, in each case degrading rather than exiting. -- Log no prompts, message bodies, model payloads, certificates, or key material. + +- Scanner runs on Windows, Linux, and macOS on x64 and arm64, without administrator + privileges. +- Request duration, response size, addresses tried, and simultaneous reads are bounded. + A slow discovered node must not block telemetry reads from verified peers. +- Scanner has few dependencies and reuses shared discovery and cluster-trust packages. +- Tests are repeatable with simulated discovery and local HTTP/TLS test servers, + without requiring real engines, multicast networks, or cluster membership. ## 6. Inputs and Outputs -**Inputs** — JSON-RPC 2.0 frames from the parent over `stdin` (or the `--ipc` endpoint), including two evidence feeds the broker relays from other workers: `discovery:set-observed-addresses` (from `nvpair-node-info`) and `discovery:node-activity` (from `nvpair-proxy`). Plus two HTTP reads the daemon performs itself: `GET /v1/node-info` on a node's `ni` port and `GET /v1/models` on its `em` port. Multicast DNS answers for `_nvpair-node._tcp` on UDP 5353 are the third input; they are self-asserted and validated only structurally. Cluster membership and the peer pin set are read from the `--cluster-dir` subtree on every refresh. +**Inputs**: broker JSON-RPC requests over stdio (standard input/output) or `--ipc`, +mDNS announcements, local identity and pins (read only), authenticated peer +node-info responses, local hardware snapshots from owned worker IPC, model lists, +and notifications of inference responses. The broker supplies local service +ports and addresses that peers have successfully used, resending registrations +after Scanner restarts. + +**Outputs**: the local mDNS record, `DirectoryNode` query results and change +notifications, node-info snapshots (measurements available at the time of a read), +smaller scheduling messages, and a startup `ready` notification. stdout carries +JSON-RPC messages written one at a time, and operational logs go to stderr. + +Directory records carry `discoveryRecordId` and `identityVerification` in addition to +the existing identity, address, service, inventory, model, and `lastSeen` fields. +`identityVerification` is `local`, `pinned`, or `unverified`. These fields record +provenance: where the data came from and how the node's identity was verified. +Scanner assigns them, rather than copying them from a remote response. + +`discoveryRecordId` identifies the record being tracked; `hostUuid` identifies +the host that record describes. For an unverified discovery, `hostUuid` is only +an advertised claim. Two announcements can claim the same `hostUuid` while +receiving different `discoveryRecordId` values because their service instances +or connection endpoints differ. Those distinct record IDs let Scanner track +the candidates separately while checking their claims. Neither identifier proves +ownership of a host; `identityVerification` records how identity was established. -**Outputs** — JSON-RPC 2.0 frames to the parent over `stdout` (or the IPC endpoint): the `ready` notification, the `discovery:node-*` event stream, the `discovery:node-telemetry` samples, and responses to requests. On the wire the daemon emits its own `_nvpair-node._tcp` record — PTR, SRV, TXT, and A records with the cache-flush bit, announced twice at startup, re-announced every 60 s and on every interface change, and retired with a TTL-0 PTR goodbye on shutdown. +## 7. API / Interface Contract -`RegisterParams` — the `discovery:register` / `discovery:update-txt` payload. Transport is deliberately absent: it is derived from the service key, never advertised. +### 7.0 Methods and notifications -```json -{ - "service": "em", - "port": 14322, - "txt": [] -} -``` +Newline-delimited JSON-RPC 2.0 over stdio by default. `--ipc ` selects a +Unix domain socket or Windows named pipe. -`DirectoryNode` — one directory entry, carried in every `discovery:node-*` event and listed in `discovery:get-nodes`. +Requests (caller → Scanner): + +| Method | Params | Result | +|---|---|---| +| `discovery:register` | `{service, port, txt?}` | `{ok:true}` | +| `discovery:unregister` | `{service}` | `{ok:true}` | +| `discovery:update-txt` | `{service, port, txt?}` | `{ok:true}` | +| `discovery:get-nodes` | `{service?}` | `{nodes:[DirectoryNode]}` | +| `discovery:reload-identity` | — | `{ok:true}` | +| `discovery:reload-trust` | — | `{ok:true}` | +| `discovery:set-observed-addresses` | `{addresses}` | `{ok:true}` | +| `discovery:node-activity` | `{hostUuid, msSince}` | `{ok:true}` | +| `log/set-level` | `{level}` | `{level}`, the accepted log level | +| `shutdown` | — | `null` | + +Registration, identity/trust reloads, observed-address updates, and activity +reports may also arrive as notifications, messages that have no request ID and +receive no reply. `shutdown` requires a request. The broker normally sends +`discovery:node-activity` as a notification when an authenticated peer has +returned inference response data. + +`discovery:update-txt` has the same handling as `discovery:register`. An empty +service name or zero port leaves the registry unchanged. Repeating a registration +or removing an absent service is also harmless: requests still return `{ok:true}`, +and unchanged registrations do not cause another network announcement. + +Malformed registration, unregistration, observed-address, and activity requests +return JSON-RPC error `-32602` (invalid parameters), without changing state. +Unknown requests return `-32601` (method not found). Notifications receive no +error reply. `log/set-level` accepts `debug`, `info`, `warn`, or `error`, and +rejects an invalid level with `-32602` when sent as a request. + +`discovery:get-nodes` filters by service when one is supplied. An absent, empty, +or unparseable filter returns all records. Results use a stable order. + +The broker also sends `discovery:local-node-info {info}` as a notification, +carrying measurements received from its own node-info worker over private +stdio. Scanner attaches these samples to its locally owned registry record, +independently of any identity or address advertised over mDNS. + +Notifications (Scanner → caller): + +| Method | Payload and meaning | +|---|---| +| `ready` | `{version}`, once at startup | +| `discovery:node-discovered` | `{node: DirectoryNode}`, a new record | +| `discovery:node-updated` | `{node: DirectoryNode}`, changed identity or inventory | +| `discovery:node-removed` | `{node: DirectoryNode}`, a record removed or replaced | +| `discovery:node-info` | `{discoveryRecordId, hostUuid, clusterUuid?, identityVerification, info}`, a full hardware snapshot from pinned mTLS or owned local IPC | +| `discovery:node-telemetry` | `{hostUuid, gpuUtilizationPercent, telemetryValid, msSince}`, GPU-usage data for scheduling | + +`discovery:node-info` sends a complete hardware snapshot for service consumers. +`info` includes GPU usage and video memory (VRAM), CPU, system memory, sample +validity, and age. Its `identityVerification` is `pinned` for an authenticated +remote source or `local` for an owned worker. Public HTTP display readings do +not enter this notification path or its caches. +Directory events report record changes, while snapshots report regular hardware +measurements. `lastSeen` remains the time the record last changed. + +`ready` is sent before network discovery starts. It reports worker startup, +not successful advertising, peer authentication, or availability of other +services. Closing the control connection or receiving an operating-system +termination signal stops the worker. A `shutdown` request is answered before +cancellation. Malformed JSON-RPC frames are logged without ending the read loop. + +Hardware sizes are in bytes and utilization is in percent. `telemetryValid` +and `msSince` describe the source GPU sample, not the age of the directory +record. When validity is false, clients ignore the age. Optional CPU, memory, +and usage fields keep their existing behavior when unavailable or omitted. + +Remote scheduling measurements are sent only for verified peers. The broker +checks that the sample still belongs to the approved peer before caching or +forwarding it to scheduler. Local samples must come from owned worker IPC. +Manually configured remote nodes meet the same pin and response-identity checks; +explicit engine configuration does not authorize using public hardware data. +Without an accepted sample, existing unavailable/stale telemetry behavior applies. + +### 7.1 Discovery records and trust + +The mDNS TXT record format stays at version 1: `uuid` (host UUID), optional +`cluster-uuid` (the certificate principal, the node identity named in its +certificate), addresses in preferred order, and registered service ports. +Node-info reports the same certificate identity as `clusterUuid`. It identifies +the node, not a shared cluster-wide identifier. Hardware and model lists are +read from the services rather than included in TXT. + +The DNS SRV (service-location) record uses the fixed port `14318`; it does not +select a service's connection port. TXT service ports identify where to try +connecting. Verified peer ports come from authenticated node-info metadata. +Unknown service keys with valid ports are retained. The TXT record lists at most +four addresses, in preferred order. + +The service instance name comes from the hostname. It can help distinguish +discovery records, but cannot establish a verified host's identity. Different +hosts may use the same name. + +Scanner assigns each unverified record a discovery record ID instead of identifying +it solely by its claimed host UUID. The canonical peer record, the main verified +record for an approved node, links its identity to its exact saved certificate. +Scanner checks a discovery claiming to be a known host against that link, even +when the announced certificate identity is missing or different. An announcement +cannot change which certificate belongs to a node or replace its verified +service addresses before authentication succeeds. + +`trusted` is true for a verified peer whose pin is still approved. For the local +record it follows actual local membership. Adding a pin schedules authentication +of matching discoveries, rather than immediately trusting their announcements. +Unverified rows cannot be merged into member rows by matching UUID, name, or +address in the broker or either frontend. + +When the user removes a peer or leaves the cluster, the affected verified peer +records disappear from the broker's desktop and TUI views. A later mDNS +announcement may appear as a separate unverified discovery. An unverified +discovery or a saved manual configuration cannot preserve the removed peer's +verification. + +### 7.2 Inventory and model transport + +Node-info serves `GET /v1/node-info` on its registered `ni` port (normally +`14318`) with a shared split listener, which accepts both HTTP and mTLS on the +same port: + +| Reader | Transport | Use of the response | +|---|---|---| +| Desktop or TUI hardware display | Public, unencrypted HTTP, including while clustered | Unverified hardware details for display only | +| Service reading an approved remote peer | Existing certificate-pinned mTLS | Authoritative peer metadata and telemetry | +| This host's services | Private broker/worker IPC | Locally owned inventory and scheduling measurements | + +Public HTTP is an ongoing frontend display interface. It is independent of the +authenticated service path, and this specification does not require moving +desktop or TUI hardware polling into the broker. Discovery and pairing use the +addresses and service ports supplied by mDNS and do not require hardware previews. +The split listener keeps public HTTP available while live membership gates the +peer node inventory handler. Neither plaintext HTTP nor TLS without a +matching peer pin can supply authenticated remote data to services. + +Local hardware reads use private IPC to keep a consistent source rule for service +measurements: local data comes from an owned worker, and remote data requires +pinned mTLS. IPC ties local measurements to the node-info process started by the +broker, rather than whichever process answers a local HTTP port. This avoids a +special exception allowing services to read hardware over HTTP only on loopback. + +Node-info publishes `nodeinfo:info-updated {info}` over its owned +stdio channel at startup and every two seconds. The broker forwards those +snapshots as `discovery:local-node-info`, adding elapsed time to sample age when +forwarding or replaying cached data. The public listener remains available to +frontend displays. A discovered record must not be treated as local merely because +its advertised address points to this computer, including a loopback address such +as `127.0.0.1` or `::1`. + +The mTLS request handler checks current membership and requires the client's +certificate to match its saved pin exactly on every request, including reused +connections. Scanner checks the server's certificate the same way, and checks +that response `hostUuid` and `clusterUuid` match the node UUID in that certificate +and the approved node identity. Keeping a certificate and private key after +leaving a cluster does not keep the authenticated endpoint available. + +Scanner uses the record's current address list for each read. It remembers a +successful address separately for node-info and engine-manager. If that address +stops answering or is removed from the list, Scanner tries the other addresses. +Among successful alternatives, it chooses the first in the record's order. +Unverified candidates cannot replace a verified peer's remembered address or +supply addresses used as evidence to choose this host's own address. + +The peer node inventory response includes the existing hardware fields and a `peer` +object containing the serving host's name, addresses in preferred order, and +registered service ports. These details come from its own local Scanner registry, +the list of services registered on that host: ```json { - "hostUuid": "5e1c9a4e-2f6b-4c8d-9a17-b3d0f4c81e22", - "name": "lab-desk-a", - "ip": "192.168.1.10", - "ips": ["192.168.1.10", "10.10.0.2"], - "clusterUuid": "a41f0c2d-7b93-4e50-8c6a-19d2e7f4b088", - "trusted": true, - "services": { - "ni": { "port": 14318 }, - "ol": { "port": 51377 }, - "wl": { "port": 14320 }, - "em": { "port": 14322 } - }, - "gpus": [ - { "name": "NVIDIA GeForce RTX 5090", "vram_bytes": 34359738368, "vram_used_bytes": 2147483648, "utilization_percent": 12 } - ], - "cpu": { "name": "AMD Ryzen 7 9800X3D", "cores": 8, "utilization_percent": 4 }, - "memory": { "total_bytes": 68719476736, "used_bytes": 21474836480 }, - "models": ["llama3:8b", "qwen:0.5b"], - "modelsByEngine": { "ollama": ["llama3:8b"], "lmstudio": ["qwen:0.5b"] }, - "loadedByEngine": { "ollama": ["llama3:8b"], "lmstudio": [] }, - "lastSeen": 1767225600 + "peer": { + "name": "workstation-b", + "addresses": ["192.0.2.20"], + "services": {"ni": 14318, "ol": 11434, "em": 14322} + } } ``` -`ip` is the node's canonical address and `ips` its full ranked list with `ip` first: for a peer, the record's own `ip=` / `ips=` followed by any other address the browse resolved for it; for this node, the locally ranked candidates. `ips` is omitted only when the node has no address at all. - -`NodeTelemetry` — the `discovery:node-telemetry` payload: - -```json -{ "hostUuid": "5e1c9a4e-2f6b-4c8d-9a17-b3d0f4c81e22", "gpuUtilizationPercent": 12, "telemetryValid": true, "msSince": 640 } -``` - -An example emitted frame — `NodeEvent` wraps the entry under `node`: - -```json -{"jsonrpc":"2.0","method":"discovery:node-discovered","params":{"node":{"hostUuid":"5e1c9a4e-2f6b-4c8d-9a17-b3d0f4c81e22","name":"lab-desk-a","ip":"192.168.1.10","trusted":false,"services":{"ni":{"port":14318}},"lastSeen":1767225600}}} -``` - -An example request and its ack: - -```json -{"jsonrpc":"2.0","id":7,"method":"discovery:register","params":{"service":"wl","port":14320}} -{"jsonrpc":"2.0","id":7,"result":{"ok":true}} -``` - -## 7. API / Interface Contract - -Three interfaces: a **local JSON-RPC interface** toward the supervising parent (newline-delimited JSON-RPC 2.0 over stdio, or a Unix socket / Windows named pipe with `--ipc`); an **mDNS interface** on UDP 5353 that advertises this node and browses for others; and an **outbound HTTP(S) client** that reads `/v1/node-info` and `/v1/models` from discovered nodes. There is no inbound HTTP listener. - -### 7.0 Methods and notifications - -Every method may arrive as a request (acked) or as a bare notification (applied silently). Unknown *requests* are rejected with `-32601` (`method not found: `); unknown notifications are logged and dropped. - -| Method | Params | Result | -|--------|--------|--------| -| `discovery:register` | `RegisterParams` — `{service, port, txt?}` | `{"ok": true}` | -| `discovery:update-txt` | `RegisterParams` — identical handling to `discovery:register` | `{"ok": true}` | -| `discovery:unregister` | `UnregisterParams` — `{service}` | `{"ok": true}` | -| `discovery:get-nodes` | `GetNodesParams` — `{service?}`; absent, empty, or unparseable returns every node | `GetNodesResult` — `{nodes: DirectoryNode[]}`, sorted by `hostUuid` | -| `discovery:reload-identity` | none | `{"ok": true}` | -| `discovery:reload-trust` | none | `{"ok": true}` | -| `discovery:set-observed-addresses` | `ObservedAddressesParams` — `{addresses: string[]}`, this host's addresses peers have reached, replacing the previous set | `{"ok": true}` | -| `discovery:node-activity` | `NodeActivityParams` — `{hostUuid, msSince}`, a peer's engine returned inference bytes `msSince` ms ago | `{"ok": true}`; the broker sends it as a notification | -| `log/set-level` | `{level}` — `debug` \| `info` \| `warn` \| `error` | `{"level": ""}` | -| `shutdown` | none | `null` | - -`discovery:node-activity` arrives while inference is streaming, so the broker sends it without an id: a request would cost a pending entry and a blocked goroutine per report for an ack nobody reads. An activity report for this node is ignored, and `msSince` is clamped to 0–60 s before use. - -Malformed `params` on `discovery:register`, `discovery:update-txt`, `discovery:unregister`, `discovery:set-observed-addresses`, or `discovery:node-activity` return `-32602` with the decode error when the frame carried an `id`, and are dropped with no reply otherwise. An invalid `log/set-level` level returns `-32602` and is warned about. A `register` / `unregister` that changes nothing is still acked `{"ok": true}` — the ack means "applied", not "changed". - -The daemon emits five notifications and never sends a request to its parent. **`discovery:node-telemetry`** carries `NodeTelemetry` for one node per successful sample and is broker-internal scheduling input, not a directory change (§7.4). **`ready`** — `{version}` — is sent once, before the mDNS responder and browser start, and is the readiness signal. **`discovery:node-discovered`**, **`discovery:node-updated`**, and **`discovery:node-removed`** each carry `NodeEvent` (`{node: DirectoryNode}`). `discovery:node-discovered` fires when an entry enters the directory; `discovery:node-updated` fires on a real change to a known entry, including a model-only or membership-only change found by the periodic sweep and a trust change found by `discovery:reload-trust`; `discovery:node-removed` fires on eviction and carries the last-known entry, and also retires a record proven superseded at its address — those removals are emitted *before* the arriving node's own event, so no consumer ever observes one machine under two identities. The one exception is the removal emitted by an identity re-stamp, which carries only `hostUuid` and `name` for the retired identity — enough for a consumer to reconcile by key. - -### 7.1 Service registration and the local record - -`registry` is this node's view of its own advertised services: a `ServiceKey → port` map plus the daemon-stamped identity. It is the only input to the advertised record. - -The canonical service keys are exactly nine, in the order they are emitted into TXT: - -| Key | Service | Port in the product | Derived transport | -|-----|---------|--------------------|-------------------| -| `ni` | `nvpair-node-info` HTTP inventory | `14318` | plain, even when clustered | -| `ol` | `nvpair-proxy` Ollama facade ingress | the facade's listen port | plain | -| `lm` | `nvpair-proxy` LM Studio facade ingress | the facade's listen port | plain | -| `lc` | `nvpair-proxy` llama.cpp facade ingress | the facade's listen port | plain | -| `er` | `nvpair-errors` cross-node ingest | `14319` | cluster mTLS when the target is clustered | -| `wl` | `nvpair-workload-manager` event ingest | `14320` | cluster mTLS when the target is clustered | -| `cl` | `nvpair-cluster-manager` pairing + trusted plane | `14321` | split channel — pairing plain, trusted endpoints mTLS | -| `em` | `nvpair-engine-manager` HTTP surface (`/v1/models`) | `14322` | plain | -| `ec` | `nvpair-engine-manager` remote control | `14323` | cluster mTLS when the target is clustered | - -Transport is **not** advertised. A consumer derives it from `ServiceKey.Transport()` plus whether the target node is clustered, because transport is global policy every same-version binary already knows rather than per-instance data. A key absent from TXT means the service is absent on that node. An unknown key parses as a service port and round-trips through the record, so a future service is carried, not dropped. - -In practice all registrations come from `nvpair-ui-broker`, which caches them (`relay.RegistrationCache`) and pushes each down as an id-bearing `discovery:register` when it starts the owning worker: `ni` when node-info spawns, `wl`, `er`, `em` likewise, `ec` only when a cluster dir is configured, `cl` when cluster-manager spawns, and `ol` / `lm` / `lc` from its engine reconcilers at the matching `nvpair-proxy` facade's listen port — never the engine's own port — with an `unregister` whenever the engine goes unhealthy or its facade is down. Because the broker owns the cache, it replays the whole set on every scanner spawn, so a scanner restart needs no reconnect logic anywhere else. - -`register` and `unregister` report whether the record actually changed. Only on a change does the daemon call `Responder.UpdateTXT` (which swaps the TXT and re-announces immediately) and refresh its own directory entry. The refresh runs on its own goroutine so a startup burst of registrations cannot serialize the read loop behind a loopback enrichment fetch. - -### 7.2 The `_nvpair-node` record and TXT layout - -This is an internal contract between the scanner daemons on a subnet. It is deliberately **not** a client contract: the desktop and `nvpair-tui` consume the broker's reported discovery snapshots and never parse mDNS. - -- **Service type**: `_nvpair-node._tcp` in domain `local`. -- **Instance name**: the OS hostname. Display only; never an identity and never unique. -- **SRV port**: the fixed constant `14318`, non-authoritative. Consumers must ignore it and read every port from TXT; it exists only so the SRV record is well-formed and deterministic. -- **TXT**: `v=1;uuid=;cluster-uuid=;ip=;ips=,…;=…`, absent fields omitted. `v` is the schema version, for diagnostics and to disambiguate future record formats — it is not a mixed-version compatibility switch. `uuid` is the stable per-host identity (`nodeid.Resolve`: cluster-manager's `cluster/identity.json` when present, else this host's `node-id.json`, else a freshly minted and persisted UUID v4). `cluster-uuid` is the cluster principal and is present only while this node is a member. `ip` is the first of `netpick.LocalCandidates`, the host's own ranked choice, which is why a consumer prefers it over the A records; `ips` is the whole ranked list, comma-separated and capped at four, and is present only when there is more than one address. -- **Address ranking** (`netpick.LocalCandidates`): loopback, link-local, and the Docker default bridge address are never published. Every other IPv4 address is ordered by, in turn: whether it *qualifies* — not on a point-to-point interface, not a /30–/32 link only the machine on its far end can reach, and not on an interface whose mDNS sends have failed for three consecutive scans; whether a remote peer has actually reached it (relayed from `nvpair-node-info` as `discovery:set-observed-addresses`); whether a discovered peer sits on its interface's subnet; whether it is the kernel's route source address; a physical-adapter preference; an address score; and finally the address string, so the order is stable across restarts. Physical interfaces lead whenever one of their addresses qualifies, and a virtual or tunnel address is then published only if a peer has reached it, and last. Unqualified addresses are still listed, after the qualified ones. At startup no peer or send evidence exists yet, which is why the order is re-ranked on every 15-second sweep and whenever the observed set changes. -- **Announcement cadence**: two announcements about a second apart at startup, then every 60 s, plus one on every observed interface-address change and one on every TXT swap. Records carry a 120-second TTL and the cache-flush bit so a peer replaces rather than accumulates. -- **Goodbye**: on context cancellation the responder multicasts a TTL-0 PTR for the instance before returning, so mDNS caches drop the record and it is absent from peers' next scan. A PAIR peer's browser does not treat the goodbye as an eviction; the absence enters the ordinary miss path (§7.3). -- **Per-interface send path**: `grandcat/zeroconf` binds one socket to the multicast group for both directions, and Windows refuses to send from a socket bound to a group — silently, because the write error is swallowed inside the library. `nvpair-shared/mdns` and `nvpair-shared/discovery` therefore keep the library's receive behavior (a joined socket receives regardless of binding) and send every announcement, reply, and PTR query from a fresh socket bound to the interface's own address and the RFC 6762 source port 5353, with an explicit multicast interface and a TTL of 255 (falling back to 1). The same path is used on every platform; the socket's reuse options let it coexist with the long-lived receive sockets and the OS's own responder, which on macOS additionally needs `SO_REUSEPORT`. Without this a multi-homed Windows host is invisible to the LAN. Each scan's per-interface send outcome also feeds the address ranking above. -- **The model list is not in TXT.** It lives behind `em`'s `/v1/models` because it can grow past the DNS per-string limit and because `nvpair-engine-manager` already owns engine state. This keeps every TXT key short and bounded. - -### 7.3 The peer directory: freshness, eviction, and goodbye - -The browser (`nvpair-shared/discovery`) is a scan-and-diff state machine over `_nvpair-node._tcp`, configured here with the shared defaults and two options: `WithKeyFunc` keying by `uuid=` and `WithLivenessProbe` wiring the daemon's `reachable`. - -- **Cadence**: one scan every 5 s, each bounded at 3 s of listening; the first scan runs immediately. -- **Change detection**: addresses and TXT are compared as multisets, because mDNS returns a multi-homed node's records in a non-deterministic order and an order-sensitive compare would emit a spurious `updated` (and its downstream churn) on every scan. -- **Eviction**: a node absent from a scan has its miss counter bumped. Two thresholds apply. From `probeAfterMisses` (3 misses, about 15 s) the liveness probe asks about the node on **every** scan, and any positive answer resets the counter and recovers the node outright. Only at `missThresholdDefault` (12 misses, a full minute) is it evicted — so eviction rests on a run of roughly ten failed probes, not on one dial that happens to land on the deadline. The window is sized for a node saturated by its own inference load, which stops answering mDNS *and* stops accepting probe connections for as long as the load runs while still serving requests. Up to 8 probes run concurrently. -- **What the probe asks**, cheapest first. A `/v1/node-info` enrichment newer than 10 s (`reachableInfoFreshness`) keeps the node with no dial. So do inference response bytes from it within the last 60 s (`activityFreshness`), reported by the broker as `discovery:node-activity` from whichever proxy saw them — the only liveness signal that gets stronger the busier the node is. Otherwise it dials the node's `ni` port and then its `em` port at each of its published addresses in order, with a 1-second timeout each, and stops at the first that connects; nothing answering is the ordinary departure and is settled without any HTTP. Inference ports (`ol` / `lm` / `lc`) are never dialed — they front the proxies, and probe traffic there contributed to accept-queue exhaustion. A node advertising neither `ni` nor `em` has nothing to probe and evicts on the threshold alone, as does one with no resolvable address. The probe runs outside the browser lock, and the node stays routable while it is in flight. -- **A scan that loses everyone**: when more than one node is known and a scan returns none of them, no node is charged a miss, for up to `emptyScanGrace` (6) consecutive scans, about 30 s; after that they count normally. Independent machines do not leave a network in the same five-second window, but a process too starved to drain its multicast socket within the scan window hears silence from all of them at once — without this one saturated host would evict its whole cluster, idle peers included, in a single reconcile. With a single known node there is no "all at once" to be suspicious of, so it falls through to the ordinary threshold. -- **A connect is not an identity**: something listening only proves *something* is at that address. A machine whose appdata was wiped mints a fresh `uuid=` while keeping its address, hostname, and fixed ports, so a connect-only verdict would reset the miss counter forever and leave the fleet carrying a permanent duplicate. When the sweep does find an address answering, `identityAlive` reads `hostUuid` from `/v1/node-info` at each published address, 1 s per address. Any address naming the record's own host keeps it; the connect is overruled only when every address that answered named someone else. It asks every address rather than only the first because one wrong answer is not proof of a wipe: a direct-connect link reuses the same subnet on other pairs of machines, so a record's leading address can resolve locally to a different host entirely. Every other answer — no `ni` advertised, no `uuid=` to compare, no response inside the 1-second budget, a response omitting `hostUuid` — is inconclusive and the connect stands. Ordering the identity read after the sweep is deliberate: it only runs when the node would otherwise have been *kept*, which is the sole case where the two need telling apart. -- **Supersedence**: the same duplicate can arrive from the other direction, as a browse event for a new identity at an address the directory already holds. `supersedeCandidates` nominates a stored record sharing the arriving node's `ip` and `name`, carrying a different `uuid=`, and at least `supersedeMinAge` (300 s) older; it deletes nothing, because all three signals come from an unauthenticated record and describe the shape of a spoof exactly as well as the shape of a wipe. Each candidate is then put to the same question at the arriving record's `ni` port, and `upsertEvicting` re-validates the verdict against the entry actually stored before removing it. An unconfirmed candidate stays and is not left uncovered — when it stops being advertised the miss-threshold path asks again. -- **Identity collisions**: when the entry under a key comes back carrying a different `uuid=`, the browser warns about a possible hostname collision or host replacement. -- **Freshness**: `lastSeen` is stamped at projection time, so it tracks the last *change* rather than the last successful scan (§4). There is no TTL-based expiry in the directory itself; the miss threshold plus the liveness probe is the whole expiry mechanism, and a goodbye simply makes the node disappear from the next scan. -- **Self**: this node's entry is registry-driven, published before the browse loop starts, and immune to browse events entirely. It is also excluded from supersedence in both directions — its `lastSeen` advances only when this node republishes, so it can look arbitrarily stale and must not be displaceable by a peer claiming its address, nor displace one. When the LAN ranker has not picked an address yet, the entry falls back to `127.0.0.1` so the local node always has a dialable host. - -### 7.4 Enrichment - -Two independent decorations, both keyed by `hostUuid` and both backed by a last-known-good cache that a transient failure falls back to. A node may advertise one without the other. - -- **Hardware** — `GET http://:/v1/node-info`, 3-second timeout, plain HTTP. The response's `GPUs`, `cpu`, and `memory` land on the entry. This stays plain **even for a clustered peer**: the daemon must never infer mTLS from a node-level `cluster-uuid=`. The only way it becomes HTTPS is the operator setting `--client-cert` / `--client-key` / `--ca-bundle`, which is global and never per node. -- **Models** — `GET /v1/models` on the `em` port, 6-second timeout, deliberately longer than engine-manager's own 5-second models sweep so a peer with several engines is not cut off mid-sweep. The body is `{"models": [...], "modelsByEngine": {...}, "loadedByEngine": {...}}`; the last two are additive, so a peer that omits them yields nil maps while the flat union still enriches the node. A present engine key with `[]` is a successful empty inventory; a missing key means that engine was not running or queryable. -- **Model transport is the cluster gate.** Which models a machine holds is cluster data, so a **peer** is fetched only over mTLS pinned to the principal it advertises. No pin, no fetch — there is deliberately no plaintext fallback for a peer. **This node's own** engine-manager is dialed over loopback in plaintext in every membership state, which is what keeps a standalone machine's own model list working. -- **Dial host**: a peer is dialed at each address it published, in its own order, until one answers for the expected `hostUuid`; an answer from a different host is skipped rather than attributed to the entry. The address that answered is remembered per node and per service (`ni` and `em` can be reachable at different addresses), and later rounds ask only it until it stops answering — otherwise a multi-homed peer whose leading address this host cannot reach would cost that address's full timeout on every sweep. This node is always dialed at `127.0.0.1`, for enrichment, the periodic refresh, and telemetry, so a first-run firewall block on inbound to its own LAN address cannot blank the local entry. -- **Node-info connections**: the node-info client holds at most one connection per origin, with a 1-second connect and TLS-handshake budget. A caller for an origin waits its turn *before* its own request timeout starts, so one waiting behind a slow but healthy response does not expire before its request is sent. A response body is drained up to 64 KiB so the connection can be reused. A peer whose node-info stops answering logs one warning, and one info line when it answers again. -- **Scheduling telemetry**: independent of enrichment, every ~2 s (a stable per-node offset within ±250 ms) the daemon GETs `/v1/node-info` from each node advertising `ni`, at most 16 in flight and one outstanding per node. It keeps only the highest `utilization_percent` across `GPUs`, `telemetryValid`, and `msSince` advanced by the request's own latency, and emits them as `discovery:node-telemetry`. A remote node whose sample fails backs off to 4, 8, 16, then 30 s and resets on success; loopback never backs off. A failure emits nothing, so downstream freshness expiry handles it; a response naming another host is never attributed. Telemetry writes nothing to the directory or the enrichment caches. -- **Periodic convergence**: every 15 s the daemon first re-ranks this host's published addresses (§7.2), then runs two passes in order, membership then models, because the model fetch is pinned mTLS keyed on the peer's principal and must dial a peer that has just joined or left under its current identity. The membership pass snapshots every peer advertising `ni` with a usable address (self excluded — `publishSelf` owns the local principal) and reads `clusterUuid` from each; the model pass snapshots every node advertising `em` with a usable address. Each pass refreshes its nodes concurrently, capped at 16 in flight and run to completion before the next tick. A model result is applied under the directory's removed-or-re-addressed guard, and the enrichment cache is written *before* the directory so a concurrent browse-driven fallback whose own fetch failed cannot revert the update with a stale cached value. Only a real change emits an event, and each pass writes only its own fields — the membership pass touches `clusterUuid` and `trusted`, the model pass the three model fields, and everything else is left to the browse path. -- **Eviction clears the caches**, so a re-discovered node re-probes from scratch rather than resurrecting stale values. - -### 7.5 Network-change handling - -`nvpair-shared/netmon` is the single watcher for host addressing, with platform backends and a 10-second polling fallback; it debounces bursts for 1500 ms and notifies only on an observed change, so a duplicated or spurious OS event produces nothing. - -Two consumers react. The **responder** refreshes its per-interface address map and re-announces with the cache-flush bit, so peers drop a stale A record promptly; it keeps the last good set rather than going dark when enumeration returns nothing, and it does not re-join the multicast group on a newly appeared interface, so queries arriving there are unanswered until restart while the periodic and on-change announcements still publish on it. The **daemon** re-runs `reloadIdentity`, which re-ranks `ip=` / `ips=` from `netpick.LocalCandidates` and re-advertises if they changed — without this the advertised address would freeze at startup, and because consumers prefer `ip=` over the live A records, a stale value would misroute them. When the monitor cannot start, the daemon logs a warning and the advertised `ip=` stays static. - -`reloadIdentity` has three triggers — the broker's explicit `discovery:reload-identity` nudge, `discovery:reload-trust` by way of `reconcileIdentity`, and a network change — and one body: re-resolve `uuid=`, refresh membership, re-rank the addresses, and return early when nothing changed. Address ranking alone also re-runs on every 15-second sweep and when the peer-observed set changes, because its strongest evidence does not exist yet at startup. When `uuid=` changed it also drops the entry under the old UUID (moving its enrichment caches to the new one), emits `discovery:node-removed` for it, and republishes self under the new identity. It is serialized against `publishSelf` by the same lock. - -### 7.6 Cluster membership and trust - -The daemon holds a live `clustertrust.Mesh` over `--cluster-dir` and re-reads it rather than snapshotting, because membership is mutable at runtime: `nvpair-cluster-manager` mints the keypair, activates an admission, and writes pins whenever the user creates, joins, or leaves a cluster — in any order relative to this process starting. - -- **Membership** is a loadable identity *and* either a live admission (`admission.json` with a non-empty `clusterId` and a non-zero epoch) or at least one pinned peer. It is deliberately **not** "do I hold a keypair": a leave or removal clears the admission and the pins but keeps `node.key` / `node.crt` so the same identity can be reused on a rejoin. Advertising on keypair presence would make a departed node look clustered to the whole fleet, so peers could never invite it back. -- **`discovery:reload-trust`** is how a pin-set change reaches the daemon. `nvpair-cluster-manager` reports `cluster:trust-changed` only after the pin is on disk and the broker relays it here, so the directory this handler reads is never older than the event that prompted it. The handler refreshes the mesh and then does two things unconditionally: `reconcileIdentity` compares live membership against the `cluster-uuid=` the registry is publishing and calls `reloadIdentity` when they disagree, and `reconcileTrust` re-derives `trusted` for every directory entry. Recomputing rather than gating on a "did it change?" answer is the point — a shared one-shot change signal is consumed by whichever caller refreshes first, which is how a membership hook can silently never fire. -- **`trusted`** is set for a browsed peer when the daemon holds a pin for the `cluster-uuid=` it advertises, refreshing the pin set first. For this node's own entry the same check applies and resolves through self-trust whenever it is clustered; self is included in the reconcile pass like any other entry. `trusted` is a local statement about pins, not a claim carried on the wire, and it is the only trust-bearing field in the directory. -- **A peer's membership converges without mDNS.** A peer's principal otherwise arrives only as `cluster-uuid=` on a record read once per browse *change*, and the liveness probe deliberately keeps a still-reachable peer's entry alive, so an unreceived re-advertisement would leave a departed peer's principal in place for the life of the process — suppressing the invite that would bring it back. The 15-second membership pass reads `clusterUuid` from each peer's own `/v1/node-info`, which under the broker is served plain and so answers whether or not the two nodes share a cluster, and reaches it over TCP at the peer's published addresses, sharing nothing with multicast reception. `applyClusterIdentity` writes the principal and the trust derived from it together, since a peer that left is no longer pinned under the principal it held. -- **What membership gates**: whether `cluster-uuid=` is advertised at all, and whether a peer's `/v1/models` is fetched. It gates nothing else — a peer with no pin is still discovered, still listed, still address-resolved, and still hardware-enriched, because `ni` is plain by policy. -- **What this daemon does not do**: no key generation, no certificate handling beyond handing a pinned config to an HTTP client, no PIN handling, no membership mutation, and no cryptography of its own. All of that is `nvpair-cluster-manager`'s. - -### 7.7 The broker relay, subscriptions, and the manual-node merge - -`discovery:subscribe` is a **broker** method, not a scanner method — the daemon neither implements nor needs it. The broker reads the daemon's delta stream on the scanner peer's read pump and folds every event into two structures: - -- `relay.Directory`, keyed by `hostUuid`, is the internal fan-out. A worker subscribes with `discovery:subscribe` and `SubscribeParams` (`{services?: ServiceKey[]}`; an empty list means all nodes) and receives `discovery:nodes` — `{nodes: DirectoryNode[]}`, its **full filtered set**, re-sent on every directory change. Consumers replace their set from each push rather than applying deltas, so a dropped or reordered push cannot leave them drifted. Today each `nvpair-proxy` engine facade subscribes for its own key — the Ollama facade for `ol`, the LM Studio facade for `lm`, the llama.cpp facade for `lc` — `nvpair-errors` for `er`, `nvpair-workload-manager` for `wl`, `nvpair-cluster-manager` for `cl`, and `nvpair-engine-manager` for `ec`. This directory is fed **only** by scanner events. -- The broker's client-facing `discoveryStore` projects each `DirectoryNode` onto its own `AvailableNode` shape and is what `discovery:get-nodes` and the `discovery:nodes-changed` push serve to the desktop and `nvpair-tui`. This is the layer clients consume; they never see `_nvpair-node`, its TXT keys, or its service keys. - -The broker also carries three feeds that are not directory events. It caches each `discovery:node-telemetry` sample as the scanner source in its telemetry cache — a scanner observation outranks the manual-node worker's for the same host — and relays the aged sample to `nvpair-job-scheduler`. It relays `nvpair-node-info`'s `nodeinfo:observed-addresses` down to this daemon as `discovery:set-observed-addresses`. And it relays each proxy's report of inference bytes from a peer down as the `discovery:node-activity` notification. - -**Manual nodes are merged there, not here.** The scanner has no manual-node concept. `nvpair-manual-nodes` probes a user-entered address, learns its real `hostUuid` from `/v1/node-info`, and the broker records that as a second *source* claim on the same `hostUuid` key. Each record tracks per-source ownership; when both claim it the scanner (mDNS) view wins, because it is the authoritative discovery record and carries `trusted`, `clusterUuid`, and per-engine models. A record is forgotten only when no source claims it, so removing a manual node never evicts a live mDNS node, and a `discovery:node-removed` that drops the scanner's claim while a manual claim survives is not treated as a departure. That final-claim distinction is what the broker uses to decide whether to fail the workloads pinned to a departed node. - -### 7.8 Versioning - -`v=1` in TXT identifies the record schema. New service keys and new `DirectoryNode` fields are **additive** and require no version bump: an unknown TXT key round-trips as a service port, and an unknown JSON field is ignored by an older consumer while `omitempty` keeps it off the wire when unset. `modelsByEngine`, `loadedByEngine`, and `ips` were added this way. A change to the meaning of an existing key, to the identity keying, or to the SRV-port-is-non-authoritative rule is a breaking change and bumps `v`; the schema is a flag-day rollout across the bundle rather than a mixed-version compatibility surface, since every binary ships together. The JSON-RPC method and notification names are frozen; new methods are additive. The binary's `main.Version` is stamped from `services/versions.json` and reported in `ready` and by `--version`. +The broker sends the local snapshot to node-info over stdio using +`nodeinfo:set-peer-metadata {hostUuid, clusterUuid, peer}`, with result +`{ok:true}`. It refreshes the snapshot after local identity, name, address, or +registration changes and resends it on restart. Peer node inventory is served +only when the snapshot agrees with the current cluster identity. Until +setup is complete, or while identity is changing, it returns `503`, while public +hardware inventory remains available. + +Peer node-info responses must fit within 65,536 bytes; Scanner rejects larger +responses. This limit applies to the authenticated service interface. + +For an approved peer, a failed pin check, TLS exchange, or identity validation +never triggers a retry over unencrypted HTTP. For public responses, absent +`clusterUuid` means unknown and an empty value is an unclustered claim, neither +of which changes an established peer's membership or verified data. + +Scanner reads a peer's models using the verified engine-manager (`em`) service +details and `GET /v1/models` over pinned mTLS. For the local host, Scanner uses +its own registry and unencrypted HTTP over loopback. It adds the combined model +list, `modelsByEngine`, and `loadedByEngine` to the directory, retaining the last +successfully read data during temporary failures. A newly authenticated peer +schedules a model refresh without waiting for the next periodic inventory sweep. + +Scanner compares model lists without depending on their order. A present engine entry +with an empty list reports no models for that engine; a missing entry means +no inventory was supplied for it. Omitted per-engine maps leave the combined +model list available without supplying per-engine details. + +Hardware and model updates preserve each other's fields. Scanner accepts a result +into records or caches only while the target record, the service's address and port, +and its approved identity are still current. A result for a removed or changed +target cannot restore old data, even if the network request succeeded. + +### 7.3 Frontend display and service decisions + +Desktop's `node-info-poller.ts` reads node-info over HTTP. TUI's +`nodetelemetry.go` also uses HTTP for node detail screens. These display reads +remain supported for local, paired, unpaired, and manually configured nodes. +They preserve GPU, VRAM, CPU, and memory displays without frontend certificate +handling or a required switch to broker-delivered hardware snapshots. + +Scanner sends its directory changes directly to the broker. `discovery:subscribe` +is a broker method, not a Scanner method. Subscribed workers receive all records +matching their requested services in `discovery:nodes`; subscribed desktop and +TUI clients receive full snapshots in `discovery:nodes-changed`. Consumers +replace their previous set with each snapshot rather than treating it as a list +of individual changes. + +Both clients continue to receive node identity, trust state, service addresses, +and model records from the broker. HTTP measurements may update hardware display +fields only. Reported identity or membership fields are unverified claims and +cannot change the broker's trusted records or be sent back as authoritative +service input. A node's identity can be verified while its HTTP display readings +remain unverified. + +Removing an unverified discovery, including when it is replaced by a verified +record, does not mean the verified peer departed. Removing a manual configuration +cannot remove an independently verified peer record. Membership or pin removal +still clears the affected verification, regardless of other discovery or manual +records that remain. + +Scanner, manual-node workers, broker caches, scheduler, proxies, and other +services must not use public node-info responses for their decisions. Remote +node-info reads use pinned mTLS; local measurements use owned worker IPC. A +manually entered address can be a connection candidate, but hardware data must +still authenticate before it is used by services. Public HTTP measurements +cannot be used to select engines, change routing, prove peer availability or +membership, or populate scheduling caches. + +An authenticated read failure leaves service data unavailable or aging according +to the existing freshness rules. It never causes a service to fall back to public +HTTP, even if a frontend can still display a public response from that node. + +### 7.4 Contract evolution + +Changes to snapshot notifications and identity-verification fields are coordinated +across shared Go types, broker forwarding and caches, proxies that read the +directory, desktop bridge, and TUI. Discovery method names and the mDNS record +format remain stable. Operator TLS settings cannot replace exact certificate-pin +checks for an approved peer. + +The TXT schema describes the advertised fields and their meanings. The new +discovery record IDs and verification fields belong to the JSON-RPC contract; +they do not change the meaning of `uuid=` or require a different TXT format. ## 8. Dependencies -**Upstream** -- A supervising parent that speaks the §7.0 contract over stdio or `--ipc`. In practice `nvpair-ui-broker`, which spawns the daemon with `--log-level` and, when configured, `--cluster-dir`; the daemon is otherwise supervisor-agnostic. -- `nvpair-cluster-manager`, indirectly and asynchronously: it writes the identity, admission record, and pins that the daemon reads from the cluster dir. The daemon works fine before it has ever run. - -**Downstream** -- `nvpair-ui-broker` — the only direct consumer of the event stream and the telemetry samples; it relays to every worker and to clients, and it feeds back the observed-address and node-activity evidence. -- Transitively, through the relay: `nvpair-proxy`, `nvpair-errors`, `nvpair-workload-manager`, `nvpair-cluster-manager`, `nvpair-engine-manager`, `nvpair-job-scheduler` (which also receives the GPU telemetry), and the desktop UI and `nvpair-tui`. -- Peer `nvpair-node-info` and `nvpair-engine-manager` instances, which this daemon reads over HTTP(S). - -**External** -- `nvpair-shared`: `mdns` (responder and per-interface sender), `discovery` (browser), `noderec` (record and RPC contract), `nodeid`, `netpick` (local ranking and remote candidates), `reach`, `netmon`, `clustertrust`, `jsonrpc`, `ipc`, `applog`, `appdir`. -- `github.com/grandcat/zeroconf` for the browse side, `github.com/miekg/dns` for record construction and the unicast query re-send, `golang.org/x/net/ipv4` for per-interface multicast control, `github.com/Microsoft/go-winio` for the Windows named-pipe transport. -- A multicast-capable IPv4 network with UDP 5353 reachable in both directions on the local subnet. +- **Broker**: `nvpair-ui-broker` starts and supervises Scanner and registers local + services. +- **Cluster manager**: `nvpair-cluster-manager` maintains node identity and trusted + certificate pins. +- **Reads data from**: node-info and engine-manager. The broker distributes its + results to frontend clients, proxies, and scheduler. +- **Shared packages**: `nvpair-shared/noderec`, discovery/address helpers, + `clustertrust`, JSON-RPC, and `applog`. No additional trust service or keys. ## 9. Data Ownership -**Owned** -- The in-memory registry of this node's advertised services and the identity stamped onto them. -- The in-memory peer directory (`hostUuid → DirectoryNode`) and its per-node enrichment caches, the remembered answering address per node and service, the telemetry retry state, the last inference-activity time per peer, and the current peer-observed address set. -- The advertised `_nvpair-node._tcp` record for this host. - -**Source of truth** -- Yes, for **network discovery from this node's perspective**: which nodes are present, their canonical dialable address, their advertised service ports, and their enriched hardware and model inventory. The broker's directory and store are projections of this stream. -- No, for node identity (`nvpair-cluster-manager` via `nodeid`), cluster membership and trust (`nvpair-cluster-manager`), hardware inventory (`nvpair-node-info`), model inventory (`nvpair-engine-manager`), or manual nodes (`nvpair-manual-nodes`). - -**Storage** -- Nothing durable is owned. The directory, the registry, and the caches are memory-only and rebuilt from scratch on every start. -- The one file the daemon can cause to be written is the fallback node id, `/node-id.json`, minted by `nodeid.Resolve` only when `cluster/identity.json` does not exist. It is written atomically at `0600` inside a `0700` directory, and it is superseded — not deleted — once cluster-manager's identity appears. -- It **reads** `/cluster/`: `identity.json`, `node.crt`, `node.key`, `admission.json`, and `trusted/.json`. All of those belong to `nvpair-cluster-manager`. -- `` is the per-user data directory every component shares: `%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows (deliberately non-roaming), `$XDG_CONFIG_HOME`/`~/.config/Nvidia Corporation/Personal AI Router` on Linux, `~/Library/Application Support/Nvidia Corporation/Personal AI Router` on macOS. `--cluster-dir` overrides the cluster subtree, and its parent then becomes the base the node id is resolved from, so both stay consistent under a custom root. +- **Owned**: the local service list, discovery directory, unverified records, + information used to choose addresses, request/retry state, and cached inventory + and measurements. These are kept in memory for the current session. +- **Source of truth**: local registry for this host's advertised services, + cluster-manager for membership/pins, node-info for hardware samples, and + engine-manager for model lists. Scanner builds its directory from these sources. +- **Storage**: Scanner reads existing identity/trust state. It adds no persistent + discovery database, pin writes, or cryptographic identity files. It uses existing + node-ID resolution, which may create `node-id.json` before cluster-manager + supplies an identity. When the local UUID changes, Scanner retires the old local + record so clients do not retain both identities. ## 10. Design Constraints -- **Performance**: one 5-second scan, one 15-second peer sweep (address re-rank, then membership, then models), a ~2-second telemetry sample of every node, and a 60-second re-announce per process. A node missing from mDNS for more than about 15 s costs up to one probe dial per address per scan. Fetches are bounded per request (3 s node-info, 6 s models) and per pass (16 concurrent), and the address memory means a steady-state round asks each node one address. Telemetry is the steady cost: every node samples every other node's `/v1/node-info` about every two seconds, which at a dozen nodes is a handful of small requests per second per host. The daemon holds no lock across a network call: the liveness probe and the supersedence proof both run outside their locks, and the enrichment caches and the directory use separate locks with a defined write order. -- **Scalability**: sized for a subnet of roughly a dozen nodes; telemetry sampling grows with the square of the node count across the fleet. The concurrency cap of 16 sits above realistic LAN counts, so it bounds a pathological fan-out rather than throttling normal operation. Directory snapshots are copied and sorted per call, which is fine at this scale and would need revisiting at hundreds of nodes. -- **Reliability**: every external dependency degrades rather than fails. No cluster dir means permanently unclustered. No network monitor means a static `ip=`. No responder means browse-only (see the §4 recovery caveat). A failed enrichment reuses the last good value. A failed announcement is retried by the next cadence tick. -- **Security**: discovery is unauthenticated by design — every TXT value is a self-assertion, and presence on `_nvpair-node` is never proof of trust. The daemon implements no cryptography: it reads `nvpair-cluster-manager`'s pins through `clustertrust` and asks it for a pinned TLS config. A peer's model inventory is fetched only over pinned mTLS, with no plaintext fallback. Hardware inventory is plain HTTP even for a clustered peer, by explicit policy, and TLS is never inferred from a node's advertised `cluster-uuid=`. The daemon exposes no listening TCP port and no inbound HTTP surface; its only bound socket is the mDNS UDP group. Private key material is read only by `clustertrust` to build a TLS config and is never logged or transmitted. -- **Compliance**: TXT and directory payloads carry opaque UUIDs, an OS hostname, LAN addresses, ports, hardware model strings, and model names — no PII, no credentials, no inference content. +- **Performance**: Scanner keeps discovery and two-second telemetry responsive by + limiting simultaneous reads. Nodes awaiting authentication have separate + identity and retry state from verified peers. Requests and response sizes are bounded. + Waiting behind another node-info read must not consume a request's own timeout + budget. Model-request deadlines allow engine-manager's bounded inventory + collection to finish. +- **Reliability**: Scanner refreshes without new mDNS announcements, tries addresses + in preferred order, and retains previously read data with its correct age. Failed + discoveries cannot replace a peer's last verified address or interrupt its reads. + Scanner reconsiders local addresses periodically and after network changes. The + broker's observed-address list, addresses that peers have reached on this host, + replaces the previous list so withdrawn observations stop influencing selection. +- **Security**: services use existing pin-based mTLS in Go. Verifying one + connection does not verify unrelated unencrypted data or every address in an + announcement. Scanner rechecks the approved node identity and exact pin before + accepting a response. +- **Transport**: one registered node-info port supports public frontend display + and peer node inventory over pinned mTLS. Public HTTP data is for display only; + service logic uses pinned remote reads or private local IPC. No additional advertised + port or frontend key handling is required. ## 11. Assumptions -- A parent supervises the process and closing its `stdin` is the shutdown signal; there is no reconnect or buffering. -- Exactly one scanner runs per host. Two would advertise conflicting records under the same instance name. -- All nodes on the subnet run the same bundle version, so the record schema is uniform (`v=1` is a diagnostic, not a compatibility switch). -- The parent registers a service only when it is actually listening, and unregisters when it stops. The daemon does not verify a registered port. -- The OS hostname is available and stable enough to serve as a display label; identity never depends on it. -- The per-user config directory is readable and writable by the running user. -- `nvpair-node-info` and `nvpair-engine-manager` serve `/v1/node-info` and `/v1/models` on the ports advertised as `ni` and `em`. -- A node's `/v1/node-info` `hostUuid` names the same identity as its advertised `uuid=`. Both eviction paths read a mismatch as a replaced machine, so a skew between the two would evict a live peer. -- Loopback is never firewalled, which is what makes self enrichment reliable on a first-run Windows host. +- The broker owns the Scanner control connection and starts/stops local workers. +- Unapproved nodes can announce themselves. They can read public hardware + inventory for display, including while the serving node is clustered. +- Authenticated peers can report their own details. Authentication proves which + node sent them, but cannot prove that a compromised approved node reports + truthful measurements. +- Local cluster state controls membership. Network reachability alone does not. ## 12. Failure Modes and Mitigations -- **This node's own multicast does not loop back** (routine on Windows): the browser ages the local node like a peer and evicts it from its own directory. → Self is registry-driven and every browse event matching this node's `uuid=` is discarded, so the local entry exists from startup and cannot be evicted by a browse. -- **First-run firewall blocks inbound to this host's own LAN address**: self enrichment fails and the local card shows no metrics and no models. → Self is always dialed at `127.0.0.1`, for the registry-driven publish, the periodic model refresh, and telemetry, while still displaying the advertised LAN address. -- **A multicast gap, transient or sustained, makes a healthy node miss scans**: it would be evicted and every consumer would drop it as a routing target. → Wait a full minute of misses before evicting, and from about 15 s in probe it on every scan: keep it on an enrichment under 10 s old, and otherwise dial its `ni` and then its `em` port at each published address and recover it on any answer. -- **A node saturated by its own inference load stops answering mDNS and probes**: its control plane is starved while it keeps serving requests, so it looks departed exactly when it is busiest, and evicting it fails its in-flight work. → The one-minute window absorbs most bursts, and inference bytes seen from it in the last 60 s keep it with no probe at all. -- **This host is too starved to read its own multicast socket**: every known node vanishes from the same scan, and charging each a miss would evict the whole cluster, idle peers included. → When two or more nodes are known and a scan returns none of them, no misses are charged for up to six consecutive scans. -- **A multi-homed peer's leading address is unreachable from here** (a direct-connect link to another machine, a VPN-only address): every fetch would spend its timeout on that address first, forever. → Walk the peer's published addresses in order, skip answers from other hosts, and remember per node and service which address answered. -- **A wiped and reinstalled node duplicates itself across the fleet**: it re-binds the same address, hostname, and fixed ports under a fresh `uuid=`, so a connect-only probe reports the pre-wipe record alive on every pass and the stale entry never ages out. → Ask the machine who it is: `identityAlive` reads `hostUuid` from `/v1/node-info` at each published address and overrules the connect only when every answer names a different host, while `supersedingUpsert` puts the same question to a candidate the arriving record is claiming the address of. Neither deletes on anything weaker. -- **An mDNS record claims a live peer's address and hostname**: treating the directory's own nomination signals as evidence would turn mDNS into a removal primitive, and since the browser re-reports a node only when its record *changes*, the evicted peer might never come back. → Nominate on address, hostname, and age; delete only on a `/v1/node-info` answer naming a different host, and re-validate that verdict against the stored entry before applying it. -- **A peer leaves without a goodbye** (crash, unplug, suspend): it lingers as an apparently valid target. → The miss threshold plus the failing liveness probe evicts it within roughly a minute; the proxies and scheduler tolerate a target that has just gone away, and its scheduler telemetry goes stale and turns neutral well before that. -- **`/v1/node-info` or `/v1/models` fetch fails transiently**: naively the entry would be blanked, flickering the node's card and its routing eligibility. → Reuse the node's last successful values; write no cache entry and emit no event on failure. A node never enriched simply stays un-enriched. -- **A `/v1/models` result lands after the node was removed or re-addressed**: it could resurrect a ghost or overwrite a moved node. → `applyModels` re-checks that the node still exists and still advertises the same `em` address and port, and discards the result otherwise. -- **A record arrives with no `uuid=`**: it cannot be keyed by a stable identity. → Warn and skip it. Keying by hostname is explicitly rejected — identity is the UUID everywhere, with no fallback. -- **Two hosts share a hostname**: they would collapse into one directory entry, merging two machines' services and models. → Key by `uuid=`, and warn when a key's identity changes. -- **The scanner starts before cluster-manager on a fresh host**: it mints and advertises its own node id, so the fleet keys on an identity that is about to change. → The broker triggers `discovery:reload-identity` once cluster-manager reports its node id; the daemon adopts the principal, moves its directory entry, and emits a removal for the stale UUID so no ghost is served. -- **This node joins or leaves a cluster while the daemon runs**: a scanner that only learned membership at startup would leave the node looking unclustered to the whole fleet — skipped by every peer's workload broadcast and dropped as a routing target while still sitting in the roster. → `discovery:reload-trust` re-derives membership from disk and re-advertises `cluster-uuid=` in place, comparing against the value the registry is publishing rather than against a change signal, so a duplicated or coalesced announcement is harmless and a missed one costs nothing beyond the next announcement. -- **A peer is pinned after this node already discovered it**: its record never moves again, so the browse-time trust annotation would stay `false` for the life of the process and both `nvpair-proxy` facades would drop it as an unpinned relay peer. → The same `discovery:reload-trust` pass re-derives `trusted` for every entry, and a removal propagates by the identical route. -- **A peer's cluster departure never reaches this host over multicast**: the stale principal outlives the cluster, and a client suppresses the invite that would bring the peer back. → The 15-second membership pass reads the peer's own `clusterUuid` from `/v1/node-info` and clears it; silence (a failed fetch, an absent field, a mismatched `hostUuid`) leaves the annotation alone rather than inferring a departure. -- **The `trusted/` directory momentarily will not read** (an antivirus lock on a freshly written pin, descriptor exhaustion): replacing the pin set with an empty one would untrust every peer for a tick and drop them all out of routing. → `clustertrust` keeps the previous set when the directory exists but cannot be read, and drops a pin only on a successful read that no longer lists it — so a teardown still propagates immediately. -- **A keypair from a past membership remains on disk after a leave or removal**: gating on keypair presence would advertise the node as clustered forever, so peers could never invite it back. → Membership requires a live admission or a pinned peer, never merely a loadable identity. -- **No multicast-capable interface at startup**: `mdns.NewResponder` fails, the node advertises nothing, `watchAddress` never starts, and nothing rebuilds the responder when an interface appears. → Logged as a warning and browse-only operation continues; recovery of the advertisement requires a worker restart, and the supervisor's restart path is the mitigation (§4). -- **The network monitor cannot start**: `ip=` would freeze at its startup value and could point at an address the host no longer holds. → Logged; the responder's own address watch still refreshes the A records, the 15-second sweep still re-ranks `ip=` / `ips=`, and an explicit `discovery:reload-identity` re-derives them. -- **A multicast-hostile segment**: nothing is discovered in either direction. → Not fixable here; the user adds the peer manually and `nvpair-manual-nodes` plus the broker's store cover it. The daemon adds no unicast discovery fallback. -- **Malformed `params` on a registration frame**: a bad frame could corrupt the advertised record. → Decode first and reject with `-32602` (when the frame is a request) before touching the registry. -- **The parent exits** (`stdin` EOF): an orphaned daemon would keep advertising a node whose services are gone. → EOF ends the read loop, which cancels the daemon context; the responder sends its goodbye and the process exits cleanly. `SIGINT` / `SIGTERM` and a `shutdown` request take the same path. +| Failure | Behavior | +|---|---| +| Missing or malformed announcement | Unusable fields are ignored; usable unverified data is retained within discovery limits | +| Peer authentication or identity mismatch | The discovery is not marked verified, the existing peer record is unchanged, and no unencrypted retry occurs | +| Inventory/model timeout or unreachable peer | Previously read inventory is retained, measurements age, and retries use increasing intervals up to the limit | +| Pin removal/replacement or local departure | Verification is cleared, affected connections close, routing stops using the peer, and scheduling samples are discarded | +| Response arriving after a trust change | The response is discarded before records are updated or telemetry is sent | +| Unverified UUID/name/address collision | Discovery and configuration records remain separate and cannot overwrite local or verified data | +| Worker restart | The broker resends local registrations/details, remote records are authenticated again, and cached sample age is preserved | +| Peer node inventory or local worker unavailable | Service measurements become unavailable/stale, no fallback to public HTTP | +| Public HTTP read fails | Frontend measurements become unavailable/stale, authenticated service reads remain independent | +| Public HTTP data enters a service cache | The data is rejected based on source, including public data attributed to manual nodes | +| Local network advertising unavailable | Scanner logs the failure and continues browsing and maintaining the local record | +| Temporary failure to read saved peer pins | A failed read does not count as removal of every peer pin | + +The local announcement follows current identity and membership. A remote peer +does not lose its approved membership merely because its authenticated endpoint +stops answering. After a removal is processed, an old response or still-open +connection cannot restore its former trust. ## 13. Observability -- **Logging**: structured `slog` on stderr with an `[nvpair-node-scanner]` prefix, level set by `--log-level` (falling back to `$NVPAIR_LOG_LEVEL`, then `info`) and mutable at runtime via `log/set-level`. At warn: mDNS advertising disabled, responder exit, browse and resolver errors, a query that left no interface, a record skipped for a missing `uuid=`, an identity change under one key, an eviction for a record superseded at its address (from either delete path, with both UUIDs), a failed node event emit, and an unavailable network monitor. At warn also: a peer's node-info that has stopped answering (once per outage) and a failed telemetry emit. At info: a peer's trust or cluster membership moving, a peer's node-info answering again, this host's published addresses being re-ranked (with the new `ip` and list), log-level changes, transport selection, and clean shutdown. At debug: per-fetch enrichment failures with the target and URL, answers skipped because a different host replied at one of a node's addresses, a liveness probe skipped for recent inference activity or kept without settling identity, model enrichment skipped because a peer is unpinned, membership skipped because a different host answered at the address, per-interface mDNS send outcomes, and re-announce decisions. Never logged: private keys, certificate PEMs, `/v1/models` or `/v1/node-info` bodies, inference content of any kind. -- **Metrics**: there is **no metrics endpoint and no metrics emission** — the daemon exposes no HTTP listener at all. Operational signals must be derived from the log stream or inferred by a parent from the event stream (directory size from `discovery:get-nodes`, churn from the `discovery:node-*` rate, enrichment health from the debug failures). Adding counters for scan outcomes, evictions, probe saves, and per-endpoint fetch success would need a new surface. -- **Alerts**: none are raised by this process. The signals worth watching in the aggregated logs are a sustained "query did not leave any interface" (multicast is broken on this host), repeated eviction and rediscovery of the same UUID (a flapping peer or an over-tight threshold), a supersedence eviction that is not explained by a reinstall (an identity skew between a node's `uuid=` and its `/v1/node-info`), "mDNS advertising disabled" at startup (this node is invisible and cannot self-heal), and a peer that stays permanently unenriched (a firewalled `ni`, or a pin this node does not hold). - -## 14. Sample usage - -The broker spawns `nvpair-node-scanner --log-level info --cluster-dir /cluster` over stdio. The daemon resolves its identity, opens a live view of the cluster dir, builds a browser for `_nvpair-node._tcp` keyed by `uuid=` with the liveness probe wired, and constructs a responder for instance `lab-desk-b` on the fixed SRV port `14318`. It emits its `ready` frame carrying the version stamped into the binary at build time, then starts the responder, the address watch, its own directory entry, the browse loop, the peer sweep, and the telemetry loop. The broker, meanwhile, replays its registration cache: `discovery:register {"service":"ni","port":14318}`, then `wl`, `er`, `em`, and `cl` as those workers come up. Each changes the record, so each triggers an `UpdateTXT` re-announce and a `publishSelf` that emits `discovery:node-updated` for `lab-desk-b` with a growing `services` map. The advertised TXT settles at `v=1;uuid=;ip=192.168.1.11;ni=14318;er=14319;wl=14320;cl=14321;em=14322`. - -On the same subnet, `lab-desk-a` powers on and announces its own record. Within 5 s the browse returns it; `lab-desk-b` has never seen this key, so the reconcile yields a `discovered` event. `toDirectoryNode` parses TXT into a `DirectoryNode` keyed by `hostUuid` `5e1c9a4e-…`, named `lab-desk-a`, addressed at its `ip=` of `192.168.1.10`, with `ni`, `ol`, `wl`, and `em` ports read from TXT and the SRV port ignored. Because the record carries a `cluster-uuid=` and `lab-desk-b` holds a pin for it, the entry is annotated `trusted: true`. Enrichment then runs twice: `GET http://192.168.1.10:14318/v1/node-info` fills in the RTX 5090, the Ryzen, and 64 GiB; `GET https://192.168.1.10:14322/v1/models` — over mTLS pinned to that principal, presenting `lab-desk-b`'s own leaf — fills in `models`, `modelsByEngine`, and `loadedByEngine`. The daemon caches both against `5e1c9a4e-…` and emits `discovery:node-discovered` with the complete entry. - -The broker's scanner read pump folds that one event into both halves of its relay: `relay.Directory.Apply` upserts the node and then re-pushes **every** subscriber its full filtered set, so the Ollama facade of `nvpair-proxy` receives a `discovery:nodes` notification listing `lab-desk-a` (it matches the `ol` filter) and immediately treats it as a routing candidate, while `nvpair-workload-manager` receives its own `wl`-filtered set. In parallel the broker's client store upserts the node under the scanner source and — because the desktop has opted in via `discovery:subscribe` — pushes a fresh `discovery:nodes-changed` array of `AvailableNode`s, and the node card appears in the UI with its GPU, its models, and a trusted badge. No client ever parsed a TXT record. - -From then on the telemetry loop samples `lab-desk-a`'s `/v1/node-info` about every two seconds at the address that answered, and emits `discovery:node-telemetry` with the 5090's utilization, `telemetryValid: true`, and the sample's age. The broker caches it and relays it to `nvpair-job-scheduler`, which uses it for GPU pressure. None of those samples touches the directory. - -Some minutes later a user pulls a model on `lab-desk-a`. Its mDNS record does not change, so no browse event fires anywhere. On the next 15-second tick `lab-desk-b` re-fetches `/v1/models`, finds one added name, refreshes its cache, applies just the model fields under the re-address guard, and emits exactly one `discovery:node-updated` — which reaches `nvpair-proxy` and the UI by the same two paths. When `lab-desk-a` is later shut down cleanly, its responder multicasts a TTL-0 goodbye; it is absent from the next scan, its telemetry samples start failing and back off, and from the third miss onward every probe of its `ni` and `em` ports is refused. At the twelfth consecutive miss, a minute after it left, `lab-desk-b` drops it, forgets its caches, and emits `discovery:node-removed`. The broker drops the scanner's claim, finds no manual claim surviving, reports the node lost so its in-flight workloads are failed, and pushes the shorter set to every subscriber. - -## 15. Process model, CLI, and build wiring - -**Process model.** A single Go binary speaking newline-delimited JSON-RPC 2.0. With no `--ipc` it uses `stdin`/`stdout` (`stdout` is reserved exclusively for protocol frames; all logging goes to `stderr`); with `--ipc ` it dials a Unix domain socket or a Windows named pipe and uses that for both directions. It runs no listener: the only bound socket is the mDNS group `224.0.0.251:5353` with `SO_REUSEADDR`, so the responder, the browser's resolver, and every other process on 5353 coexist. Every other network operation is an outbound dial. In the product the broker is the parent; the binary is supervisor-agnostic and the parent hides the console window on Windows. - -**Shutdown** has three equivalent triggers: `stdin` EOF (the parent closed the pipe), `SIGINT` / `SIGTERM`, and a `shutdown` request — which is answered before the cancel so the parent sees the reply. All three cancel the root context, which stops the browse loop, the peer sweep, the telemetry loop (draining its in-flight samples), the address watch, and the responder; the responder sends its goodbye on the way out. A JSON-RPC read error that is not EOF is logged and the loop continues, so one malformed frame does not kill the process. A fatal startup problem — an invalid TLS flag combination, an unloadable cert or CA bundle, or an unreachable `--ipc` endpoint — exits non-zero before any advertising begins. - -**CLI flags** - -| Flag | Default | Meaning | -|------|---------|---------| -| `--ipc ` | _(stdio)_ | Unix socket or Windows named pipe instead of `stdin`/`stdout`. | -| `--cluster-dir ` | _(none)_ | Cluster subtree (`node.crt`/`node.key`, `admission.json`, `trusted/`). Enables `cluster-uuid=` advertising, the `trusted` annotation, and peer model fetches; its parent becomes the base the node id resolves from. | -| `--client-cert ` | _(none)_ | PEM client certificate for node-info enrichment over HTTPS. Requires `--client-key`. | -| `--client-key ` | _(none)_ | PEM private key matching `--client-cert`. Requires `--client-cert`. | -| `--ca-bundle ` | _(none)_ | PEM CA bundle for verifying node-info server certificates, added on top of the system trust store. | -| `--log-level ` | `info` | `debug` \| `info` \| `warn` \| `error`; falls back to `$NVPAIR_LOG_LEVEL`. Mutable at runtime via `log/set-level`. | -| `--version` | | Print `main.Version` and exit. | - -Setting any of the three TLS flags moves node-info enrichment onto HTTPS; setting none keeps it plain, which is the shipped configuration — the broker passes only `--log-level` and, when configured, `--cluster-dir`. `--client-cert` and `--client-key` must be given together or not at all; `--ca-bundle` alone means "trust these roots but present no client identity". No flag carries the node identity or the cluster identity, so the binary self-bootstraps under any supervisor. +- **Logging**: logs contain structured operational details on stderr, including + service, node or discovery record ID, authentication outcome, retries, and + directory changes. Key material, certificates, pairing PINs, prompts, and + response bodies are never logged. +- **Freshness**: snapshots report whether samples are usable and how old they are. + Resending a sample never reduces its age. Failed reads must not make a node + appear freshly measured with zero usage. +- **Events**: clients receive notifications when directory data changes. Seeing + the same announcement or reloading unchanged trust data does not send another update. + +## 14. Example runtime flow + +The broker starts Scanner, registers the local service ports, and receives its +directory and telemetry notifications. Scanner announces this host and listens +for other nodes on the local network. Unpaired nodes appear in discovery rows. +Approved peers are authenticated before their details enter verified records. + +```mermaid +sequenceDiagram + participant B as Broker + participant S as Node Scanner + participant P as Network node + participant U as Desktop and TUI + participant Q as Scheduler + B->>S: Register local services + B->>S: Local worker measurements over private IPC + S-->>P: Advertise local node over mDNS + P-->>S: Node announcement over mDNS + alt Approved cluster peer + S->>P: GET node-info with pinned mTLS + P-->>S: Authenticated identity, services and hardware + S->>P: GET models with pinned mTLS + P-->>S: Available and loaded models + S-->>B: Verified node details and measurements + B-->>Q: Verified GPU usage + else Unpaired node + S-->>B: Unverified discovery record + end + B-->>U: Node identity, trust and service addresses + U->>P: GET public node-info over HTTP + P-->>U: Hardware measurements for display only +``` -**Build wiring.** `nvpair-node-scanner` is one of the twelve shipped binaries. Its version lives at `components.nvpair-node-scanner` in `services/versions.json` and is stamped in with `-ldflags "-X main.Version=…"`; it is never hardcoded. `services/build.sh` and `services/build.bat` each build it and stage it into `services/build/bin/`, and `desktop` builds the same tree into `desktop/cli-bin/` via `npm run build:modular-binaries`. It is installed by `services/installer/nvpair-setup.nsi`, listed in `services/bom.md`, and — because it is the host's mDNS listener — needs **two** Windows firewall rules that the installer adds and removes as a pair: a program rule for the executable and an inbound `UDP 5353` rule scoped to `localsubnet`. It needs no TCP rule, since it listens on no TCP port. The desktop side marks it `needsFirewallAccess: true` in `desktop/src/shared/constants/modular-binaries.ts` with `launchOwner: 'broker'`, and `npm run service-contracts:check` verifies that flag against the macOS privileged-helper list. +The diagram shows separate display and service paths. HTTP display readings +never flow into scheduler or verified peer records. Service inventory refreshes +every 15 seconds and measurements every two seconds, independently of mDNS +changes. The broker forwards trust changes to Scanner and clears affected +verification and service samples when membership or pins change. diff --git a/services/nvpair-tui/ui/nodeswire.go b/services/nvpair-tui/ui/nodeswire.go index 6721b91c8..a79bb72bc 100644 --- a/services/nvpair-tui/ui/nodeswire.go +++ b/services/nvpair-tui/ui/nodeswire.go @@ -12,10 +12,12 @@ import "nvpair-shared/noderec" // availableNode mirrors the broker's discovery boundary shape (the // discovery:get-nodes element and discovery:nodes-changed payload entry). type availableNode struct { - ID string `json:"id"` - HostUUID string `json:"hostUuid"` - Name string `json:"name"` - IPAddress string `json:"ipAddress"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + ID string `json:"id"` + HostUUID string `json:"hostUuid"` + Name string `json:"name"` + IPAddress string `json:"ipAddress"` // IPAddresses is every address the node published, in its own ranked order // with IPAddress first. A multi-homed node has no single address every peer // can reach — a direct-connect link only works from the machine on its far diff --git a/services/nvpair-tui/ui/nodeswire_test.go b/services/nvpair-tui/ui/nodeswire_test.go new file mode 100644 index 000000000..f80b6499c --- /dev/null +++ b/services/nvpair-tui/ui/nodeswire_test.go @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "nvpair-shared/noderec" +) + +func TestAvailableNode_DecodesBrokerIdentityFields(t *testing.T) { + const body = `{"hostUuid":"host-a","discoveryRecordId":"observation:candidate-a","identityVerification":"unverified"}` + var node availableNode + require.NoError(t, json.Unmarshal([]byte(body), &node)) + assert.Equal(t, "observation:candidate-a", node.DiscoveryRecordID) + assert.Equal(t, noderec.IdentityUnverified, node.IdentityVerification) +} diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 37f8fc0d7..40f72466e 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -197,6 +197,12 @@ Emitted once on startup, after the scanner (and, when available, node-info and o **Opt-in.** Only delivered to a peer that has called `discovery:subscribe`; until then (and after `discovery:unsubscribe`) the stream is silent. Once subscribed, it's fired every time the discovery store mutates — that is, whenever the daemon reports a `discovery:node-discovered`, `discovery:node-updated`, or `discovery:node-removed` (or a manual node changes). The payload is the full current snapshot as a **bare array** of `AvailableNode` objects (same shape as the array element in `discovery:get-nodes`), sorted by `id`. +Snapshots retain scanner-assigned `discoveryRecordId` and `identityVerification` +(`local`, `pinned`, or `unverified`). Manual configuration records have their own +discovery record IDs and are `unverified`. These fields describe provenance separately +from the `trusted` pin annotation; a claimed UUID or an existing pin alone does +not establish authenticated endpoint ownership. + ```json { "jsonrpc":"2.0", diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 5f3b85e4a..b62d76529 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -70,8 +70,10 @@ type VersionResult struct { // the rest of the NVPAIR codebase uses snake_case, so this is the // deliberate camelCase island at the broker's outer boundary. type AvailableNode struct { - ID string `json:"id"` - Name string `json:"name"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + ID string `json:"id"` + Name string `json:"name"` // HostUUID is the node's stable per-host identity (the daemon's uuid= TXT). // It is invariant across a PC rename and unique per physical machine, so it // is the key clients should dedup/track nodes by; id/name stay the hostname diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 8f2165336..48048a2e5 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -34,7 +34,9 @@ type ( ) type EnrichedNode struct { - ID string `json:"id"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + ID string `json:"id"` // HostUUID is the node's stable per-host identity (the daemon's uuid= TXT). // It's the discovery-store key, so a PC rename — which changes the hostname // (ID) but not the UUID — updates the existing entry in place instead of @@ -95,20 +97,22 @@ func directoryToEnriched(n noderec.DirectoryNode) EnrichedNode { port = ni.Port } return EnrichedNode{ - ID: n.Name, - HostUUID: n.HostUUID, - Host: n.Name, - Port: port, - Addresses: addrs, - TXT: txt, - GPUs: n.GPUs, - CPU: n.CPU, - Memory: n.Memory, - Trusted: n.Trusted, - Clustered: n.Clustered(), - Models: n.Models, - ModelsByEngine: n.ModelsByEngine, - LoadedByEngine: n.LoadedByEngine, + DiscoveryRecordID: n.DiscoveryRecordID, + IdentityVerification: n.IdentityVerification, + ID: n.Name, + HostUUID: n.HostUUID, + Host: n.Name, + Port: port, + Addresses: addrs, + TXT: txt, + GPUs: n.GPUs, + CPU: n.CPU, + Memory: n.Memory, + Trusted: n.Trusted, + Clustered: n.Clustered(), + Models: n.Models, + ModelsByEngine: n.ModelsByEngine, + LoadedByEngine: n.LoadedByEngine, } } @@ -311,18 +315,20 @@ func (sn storedNode) toAvailable() AvailableNode { candidates = nil } return AvailableNode{ - ID: n.ID, - Name: n.ID, - HostUUID: n.HostUUID, - IPAddress: primary, - IPAddresses: candidates, - Port: n.Port, - LastSeen: sn.lastSeen.Unix(), - Trusted: n.Trusted, - Clustered: n.Clustered, - Models: n.Models, - ModelsByEngine: n.ModelsByEngine, - LoadedByEngine: n.LoadedByEngine, + DiscoveryRecordID: n.DiscoveryRecordID, + IdentityVerification: n.IdentityVerification, + ID: n.ID, + Name: n.ID, + HostUUID: n.HostUUID, + IPAddress: primary, + IPAddresses: candidates, + Port: n.Port, + LastSeen: sn.lastSeen.Unix(), + Trusted: n.Trusted, + Clustered: n.Clustered, + Models: n.Models, + ModelsByEngine: n.ModelsByEngine, + LoadedByEngine: n.LoadedByEngine, } } diff --git a/services/nvpair-ui-broker/discovery_test.go b/services/nvpair-ui-broker/discovery_test.go index 39362b3c9..8caec6200 100644 --- a/services/nvpair-ui-broker/discovery_test.go +++ b/services/nvpair-ui-broker/discovery_test.go @@ -270,3 +270,23 @@ func TestScannerProcess_RoutesAndRemovesTelemetry(t *testing.T) { sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, "peer", "peer-uuid")) require.Equal(t, "peer-uuid", removed, "removed telemetry host") } + +func TestDiscoveryStore_SnapshotPreservesScannerIdentityFields(t *testing.T) { + const body = `{"hostUuid":"host-a","name":"workstation","discoveryRecordId":"observation:candidate-a","identityVerification":"unverified"}` + var node noderec.DirectoryNode + require.NoError(t, json.Unmarshal([]byte(body), &node)) + store := newDiscoveryStore() + store.Upsert(directoryToEnriched(node), sourceScanner) + snapshot := store.Snapshot() + require.Len(t, snapshot, 1) + wire, err := json.Marshal(snapshot[0]) + require.NoError(t, err) + type nodeIdentityFields struct { + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification string `json:"identityVerification"` + } + var got nodeIdentityFields + require.NoError(t, json.Unmarshal(wire, &got)) + assert.Equal(t, "observation:candidate-a", got.DiscoveryRecordID) + assert.Equal(t, "unverified", got.IdentityVerification) +} diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index d131f6c67..83863167b 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -237,15 +237,17 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { hostUUID = s.ID } en := EnrichedNode{ - ID: s.ID, - HostUUID: hostUUID, - Host: s.Address, - Port: s.NodeInfoPort, - GPUs: s.GPUs, - CPU: s.CPU, - Memory: s.Memory, - Models: mergeModels(s.OllamaModels, s.LMStudioModels), - ModelsByEngine: manualModelsByEngine(s), + DiscoveryRecordID: "manual:" + s.ID, + IdentityVerification: noderec.IdentityUnverified, + ID: s.ID, + HostUUID: hostUUID, + Host: s.Address, + Port: s.NodeInfoPort, + GPUs: s.GPUs, + CPU: s.CPU, + Memory: s.Memory, + Models: mergeModels(s.OllamaModels, s.LMStudioModels), + ModelsByEngine: manualModelsByEngine(s), } if s.Address != "" { en.Addresses = []string{s.Address} diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index f44a263d6..48accee12 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -474,6 +474,26 @@ type ServiceStatus struct { Probe ProbeStatus `json:"probe,omitempty"` } +// IdentityVerification records how the scanner established a record's identity. +type IdentityVerification string + +const ( + // IdentityLocal is assigned to this host's record published from the owned + // local registry, for example when scanner publishes registered local services. + // A discovered record claiming our UUID or a loopback address is not local. + IdentityLocal IdentityVerification = "local" + + // IdentityPinned is assigned after a remote node-info read authenticates the + // endpoint against the exact currently approved certificate pin and validates + // the response identity, for example when promoting a discovered cluster peer. + IdentityPinned IdentityVerification = "pinned" + + // IdentityUnverified is assigned when endpoint ownership has not been proved, + // for example for an mDNS announcement or a manually configured address before + // authentication. This includes announcements claiming an identity with a pin. + IdentityUnverified IdentityVerification = "unverified" +) + // DirectoryNode is one node in the consolidated directory served by // discovery:get-nodes, carried in the daemon's discovery:node-* events, and // listed in the broker's discovery:nodes snapshots. It replaces @@ -481,9 +501,24 @@ type ServiceStatus struct { // derived from Services + Clustered (never carried), and node-info enrichment // decorates entries that advertise a node-info service. type DirectoryNode struct { - HostUUID string `json:"hostUuid"` - Name string `json:"name"` // mDNS instance name (hostname) - IP string `json:"ip"` // canonical dialable LAN address + // DiscoveryRecordID is PAIR's tracking key for one mDNS discovery, local + // registry record, or manual configuration. HostUUID is the machine UUID + // that discovery or configuration *claims* to represent. + // + // For example, an announcement at 192.0.2.20 and an impersonating announcement + // at 192.0.2.30 can both advertise HostUUID="host-a". The scanner assigns them + // different DiscoveryRecordIDs because their connection candidates differ. + // The advertised HostUUID alone cannot distinguish those announcements. + // + // A separate DiscoveryRecordID means a separate candidate to track, not proof + // of a different physical machine. IdentityVerification records whether the + // host identity comes from our owned local registry, pinned authentication, + // or an unverified claim. + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification IdentityVerification `json:"identityVerification"` + HostUUID string `json:"hostUuid"` + Name string `json:"name"` // mDNS instance name (hostname) + IP string `json:"ip"` // canonical dialable LAN address // IPs is every address this node published, in its own ranked order with IP // first. A multi-homed node has no single address every peer can reach — a // direct-connect link is reachable only from the machine on its far end — so diff --git a/services/shared/noderec/noderec_test.go b/services/shared/noderec/noderec_test.go index fc31703e3..62414cf2f 100644 --- a/services/shared/noderec/noderec_test.go +++ b/services/shared/noderec/noderec_test.go @@ -183,3 +183,34 @@ func TestValidateTXTSize_RejectsOversizedEntries(t *testing.T) { big := "x=" + strings.Repeat("m", 300) assert.Error(t, ValidateTXTSize([]string{"v=1", big}), "oversized TXT entry not flagged") } + +func TestDirectoryNode_JSONRoundTripPreservesIdentityFields(t *testing.T) { + type nodeIdentityFields struct { + HostUUID string `json:"hostUuid"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification string `json:"identityVerification"` + } + + test := func(name, discoveryRecordID, verification string) { + t.Run(name, func(t *testing.T) { + identity := nodeIdentityFields{ + HostUUID: "host-a", + DiscoveryRecordID: discoveryRecordID, + IdentityVerification: verification, + } + body, err := json.Marshal(identity) + require.NoError(t, err) + var node DirectoryNode + require.NoError(t, json.Unmarshal(body, &node)) + body, err = json.Marshal(node) + require.NoError(t, err) + var got nodeIdentityFields + require.NoError(t, json.Unmarshal(body, &got)) + assert.Equal(t, identity, got) + }) + } + + test("owned local record", "local:host-a", "local") + test("authenticated peer record", "peer:host-a", "pinned") + test("unverified discovery record", "observation:candidate-a", "unverified") +} From 60f7ffb15cb1b4c3071cd6ee637a6e22c7e3c062 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Mon, 19 Oct 2026 11:01:28 -0400 Subject: [PATCH 2/8] Add owned node-info inventory IPC Suggested review order: - services/nvpair-node-scanner/spec.md and services/nvpair-node-info/spec.md - reference contracts for local ownership and peer metadata. - docs/architecture.mdx - services/shared/noderec/noderec_test.go, then noderec.go; services/nvpair-ui-broker/telemetry.go - sample age includes cache and relay time, with broker replay reusing the shared aging helper. - services/nvpair-node-info/main_test.go, then main.go - the owned hardware producer and peer-metadata input. - services/nvpair-ui-broker/nodeinfo_test.go, then nodeinfo.go - stdinMu orders snapshots without holding the metadata cache lock during pipe I/O. - services/nvpair-ui-broker/discovery.go and broker.go - forwarding owned snapshots and local registry metadata. - services/nvpair-node-scanner/directory_test.go, then directory.go - atomic hardware updates preserve concurrent model refreshes. - services/nvpair-node-scanner/daemon_test.go, then scanner.go and daemon.go - receiving and applying local inventory. - Node-info, scanner, and broker READMEs, then desktop/docs/service-contract-exceptions.json. Signed-off-by: Kaylee Lubick --- desktop/docs/service-contract-exceptions.json | 1 + docs/architecture.mdx | 10 + services/nvpair-node-info/README.md | 32 +- services/nvpair-node-info/main.go | 77 +++- services/nvpair-node-info/main_test.go | 257 ++++++++++++++ services/nvpair-node-scanner/README.md | 18 +- services/nvpair-node-scanner/daemon.go | 101 ++++-- services/nvpair-node-scanner/daemon_test.go | 142 ++++---- services/nvpair-node-scanner/directory.go | 42 ++- .../nvpair-node-scanner/directory_test.go | 23 ++ services/nvpair-node-scanner/scanner.go | 24 +- services/nvpair-ui-broker/README.md | 14 + services/nvpair-ui-broker/broker.go | 42 ++- services/nvpair-ui-broker/discovery.go | 26 +- services/nvpair-ui-broker/nodeinfo.go | 148 +++++++- services/nvpair-ui-broker/nodeinfo_test.go | 330 ++++++++++++++++++ services/nvpair-ui-broker/telemetry.go | 21 +- services/shared/noderec/noderec.go | 65 ++++ services/shared/noderec/noderec_test.go | 23 ++ 19 files changed, 1212 insertions(+), 184 deletions(-) create mode 100644 services/nvpair-node-info/main_test.go create mode 100644 services/nvpair-ui-broker/nodeinfo_test.go diff --git a/desktop/docs/service-contract-exceptions.json b/desktop/docs/service-contract-exceptions.json index 0f0d36602..eaa69e91b 100644 --- a/desktop/docs/service-contract-exceptions.json +++ b/desktop/docs/service-contract-exceptions.json @@ -1,6 +1,7 @@ { "$comment": "Intentionally unintegrated JSON-RPC methods for desktop ↔ services contract checks. Each ignoredMethods entry needs a current technical reason. Edit by hand; run npm run service-contracts:write to refresh docs/services-api.md after services/ changes.", "ignoredMethods": { + "nodeinfo:info-updated": "Owned node-info worker inventory, consumed by Broker.acceptLocalInfo and relayed to its scanner as discovery:local-node-info. Desktop and TUI retain their independent public HTTP hardware display path.", "engine:settings-projection": "Broker-to-engine-manager full settings projection for the pinned peer subscription hub. Consumed by Manager.handleSettingsMessage; the desktop consumes engine:settings-changed instead.", "engine:settings-reply": "Correlated broker-to-engine-manager reply to the pinned peer settings relay and the configure operation's narrow proxy rebind callback. Consumed by settingsRelay.reply, never a renderer event.", "engine:settings-request": "Engine-manager-to-broker correlated request for settings get/preview/apply or a validated operation's proxy rebind. Consumed by Broker.handleSettingsRelay.", diff --git a/docs/architecture.mdx b/docs/architecture.mdx index ffbe2fa5e..dcb2084f8 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -234,6 +234,16 @@ shows: Each of these listens on its own port. Refer to the [port map](#port-map) below. +Local service inventory and scheduling samples arrive from the owned node-info +worker over private stdio IPC, through the broker to scanner. IPC ties these +measurements to the worker the broker started, rather than whichever process +answers a local HTTP port. This works before joining and after leaving a cluster. +The broker retains the latest snapshot for scanner restart replay and adds +cache/queue time to its sample age so replay cannot make an old sample appear +fresh. Scanner's owned local registry supplies node-info's peer metadata over the +reverse IPC path; discovered announcements cannot supply local measurements or +metadata. + Electron polls every discovered node's `/v1/node-info` directly because that detail is richer than the broker's control surface exposes. Healthy nodes are sampled every two seconds; consecutive failures back off to 4, 8, 16, then 30 diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index 8ed76948d..2b2484cc1 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -12,9 +12,35 @@ A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical Two surfaces: - **HTTP(S)** — serves the node inventory at `/v1/node-info`. Plaintext HTTP on `:14318` by default; optional HTTPS (with optional mTLS) on `:14319` when a cert/key pair is supplied. -- **stdio JSON-RPC 2.0** — newline-delimited, used for lifecycle/control (`log/set-level`, `nodeinfo:set-cluster-identity`) and shutdown via stdin EOF. The service is normally launched as a subprocess by the broker. - -It does **not** advertise itself over mDNS. Discovery is centralized in the `nvpair-node-scanner` daemon: the broker registers this service's `ni` port with the daemon, which carries it on the node's one `_nvpair-node` record and fetches `/v1/node-info` over plain HTTP to enrich each node. +- **stdio JSON-RPC 2.0** — newline-delimited, used for local inventory snapshots, registry metadata, lifecycle/control, and shutdown via stdin EOF. The service is normally launched as a subprocess by the broker. + +It does **not** advertise itself over mDNS. Discovery is centralized in the +`nvpair-node-scanner` daemon: the broker registers this service's `ni` port on +the node's one `_nvpair-node` record. Remote scanners currently read its public +HTTP inventory; the owned local scanner receives private IPC snapshots. + +### Private inventory IPC + +`nodeinfo:info-updated {info}` publishes the hardware response at startup and +every two seconds over stdout. The broker relays it as +`discovery:local-node-info {info}` to its owned scanner for local inventory and +scheduling. Local service measurements work before joining and after leaving a +cluster, without a loopback HTTP read. Public HTTP hardware displays are unchanged. +The broker adds cache and queue time to `info.msSince` on forwarding or replay; +invalid samples remain invalid. + +`nodeinfo:set-peer-metadata {hostUuid, clusterUuid, peer}` accepts the local +scanner registry snapshot over stdin, with `{ok:true}` for an id-bearing request. +`peer` contains `{name, addresses, services}`; `services` maps service keys to +ports. A mismatched host identity or malformed payload returns invalid params +(`-32602`). Each accepted snapshot replaces the previous one completely, so a +removed service or address is cleared. An empty `clusterUuid` explicitly clears +membership. The broker refreshes this metadata on local registry changes and +resends it after node-info restarts. Metadata is retained for the protected +inventory handler described in scanner's spec; it is not added to public HTTP. + +`nodeinfo:set-cluster-identity {clusterUuid}` continues to update the public +membership field. `log/set-level` is also accepted. ## CLI Flags diff --git a/services/nvpair-node-info/main.go b/services/nvpair-node-info/main.go index 0b1f34c07..96f294a48 100644 --- a/services/nvpair-node-info/main.go +++ b/services/nvpair-node-info/main.go @@ -22,6 +22,7 @@ import ( "nvpair-shared/applog" "nvpair-shared/clustertrust" + "nvpair-shared/jsonrpc" "nvpair-shared/nodeid" "nvpair-shared/noderec" "nvpair-shared/splitlisten" @@ -147,6 +148,70 @@ func handleClusterIdentity(msg applog.StdinMessage, identity *clusterIdentity) { slog.Info("cluster identity updated", "clustered", params.ClusterUUID != "") } +// peerMetadata holds complete, immutable snapshots supplied by the broker from +// scanner's owned local registry. Public HTTP responses never populate it. +type peerMetadata struct { + // Readers must see a complete snapshot rather than partially replaced fields. + mu sync.RWMutex + value noderec.PeerMetadataParams + known bool +} + +// get returns the latest registry snapshot and whether the broker supplied one. +func (m *peerMetadata) get() (noderec.PeerMetadataParams, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.value, m.known +} + +// handlePeerMetadata replaces registry metadata from the broker, rejecting +// malformed payloads or a different host identity. Id-bearing requests receive +// an acknowledgement or invalid-params error. +func handlePeerMetadata(msg applog.StdinMessage, hostUUID string, metadata *peerMetadata, out *applog.Notifier) { + if msg.Method != noderec.MethodSetPeerMetadata { + return + } + var params noderec.PeerMetadataParams + err := json.Unmarshal(msg.Params, ¶ms) + valid := err == nil && hostUUID != "" && params.HostUUID == hostUUID + if valid { + metadata.mu.Lock() + metadata.value, metadata.known = params, true + metadata.mu.Unlock() + } + if len(msg.ID) == 0 { + return + } + response := jsonrpc.Message{JSONRPC: "2.0", ID: &msg.ID} + if valid { + response.Result = json.RawMessage(`{"ok":true}`) + } else { + response.Error = &jsonrpc.RPCError{Code: -32602, Message: "invalid local peer metadata"} + } + if err := json.NewEncoder(out).Encode(response); err != nil { + slog.Warn("failed to acknowledge peer metadata", "err", err) + } +} + +type infoUpdateParams struct { + Info json.RawMessage `json:"info"` +} + +// publishInfoUpdates sends inventory directly to the owning broker at startup +// and on each tick until cancellation, using the same builder as public HTTP. +func publishInfoUpdates(ctx context.Context, out *applog.Notifier, body func() []byte, ticks <-chan time.Time) { + for { + if err := out.Notify(noderec.NotifyInfoUpdated, infoUpdateParams{Info: body()}); err != nil { + slog.Warn("failed to publish local inventory", "err", err) + } + select { + case <-ctx.Done(): + return + case <-ticks: + } + } +} + // detectGPUs lives in gpu_windows.go (DXGI), gpu_linux.go (nvidia-smi with a // ghw fallback), gpu_darwin.go (IORegistry), and gpu_other.go (ghw fallback). // detectCPU lives in cpu_detect.go; memory detection is split so macOS can use @@ -407,10 +472,11 @@ func main() { return &principal } - mux := http.NewServeMux() - mux.HandleFunc("/v1/node-info", nodeInfoHandler(mesh, func() []byte { + responseBody := func() []byte { return buildResponse(gpus, cpu, memTotal, collector.Snapshot(), hostUUID, clusterPrincipal()) - })) + } + mux := http.NewServeMux() + mux.HandleFunc("/v1/node-info", nodeInfoHandler(mesh, responseBody)) // Listener layout (set up below depending on flags). Exactly one of the two // blocks that own *port runs: @@ -546,9 +612,14 @@ func main() { // interleave. notifier := applog.NewNotifier(os.Stdout) go observer.reportLoop(ctx.Done(), notifier) + infoTicks := time.NewTicker(2 * time.Second) + defer infoTicks.Stop() + go publishInfoUpdates(ctx, notifier, responseBody, infoTicks.C) + metadata := &peerMetadata{} go applog.StdinRPC(notifier, func(msg applog.StdinMessage) { handleClusterIdentity(msg, identity) + handlePeerMetadata(msg, hostUUID, metadata, notifier) }, func() { log.Print("stdin closed, shutting down") cancel() diff --git a/services/nvpair-node-info/main_test.go b/services/nvpair-node-info/main_test.go new file mode 100644 index 000000000..fc3ea456c --- /dev/null +++ b/services/nvpair-node-info/main_test.go @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "nvpair-shared/applog" + "nvpair-shared/jsonrpc" + "nvpair-shared/noderec" +) + +func TestNodeInfo_StreamsPrivateSnapshotsAndAcceptsMetadata(t *testing.T) { + // Exercise the real node-info process, with this test acting as its broker. + // Verify that unsolicited snapshots reflect membership updates, metadata + // requests are acknowledged, and public HTTP stays available without + // exposing the peer metadata. + const hostUUID = "owned-host" + const clusterPrincipal = "principal" + const peerName = "workstation" + const peerAddress = "192.0.2.20" + + process := startNodeInfoForTest(t, hostUUID) + + assert.Nil(t, process.nextSnapshot(t).ClusterUUID) + // The next snapshot must arrive without any HTTP request or IPC trigger. + process.nextSnapshot(t) + + metadata := noderec.PeerMetadataParams{ + HostUUID: hostUUID, + Peer: noderec.PeerMetadata{ + Name: peerName, + Addresses: []string{peerAddress}, + Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: process.port}, + }, + } + require.True(t, process.requestPeerMetadata(t, metadata).OK) + + require.NoError(t, process.codec.Notify(noderec.MethodSetClusterIdentity, noderec.ClusterIdentityParams{ClusterUUID: clusterPrincipal})) + info := process.nextSnapshot(t) + require.NotNil(t, info.ClusterUUID) + require.Equal(t, clusterPrincipal, *info.ClusterUUID) + require.NoError(t, process.codec.Notify(noderec.MethodSetClusterIdentity, noderec.ClusterIdentityParams{})) + info = process.nextSnapshot(t) + require.NotNil(t, info.ClusterUUID) + assert.Empty(t, *info.ClusterUUID) + + client := &http.Client{Timeout: 2 * time.Second} + t.Cleanup(client.CloseIdleConnections) + response, err := client.Get(process.inventoryURL) + require.NoError(t, err) + defer response.Body.Close() + type publicInventory struct { + HostUUID string `json:"hostUuid"` + GPUs json.RawMessage `json:"GPUs"` + Peer *noderec.PeerMetadata `json:"peer,omitempty"` + } + var public publicInventory + require.NoError(t, json.NewDecoder(response.Body).Decode(&public)) + assert.Equal(t, http.StatusOK, response.StatusCode) + assert.Equal(t, hostUUID, public.HostUUID) + assert.NotNil(t, public.GPUs) + assert.Nil(t, public.Peer) +} + +// nodeInfoTestProcess is the test's broker-side connection to a real child. +type nodeInfoTestProcess struct { + stdin io.Writer + codec *jsonrpc.Codec + hostUUID string + port int + inventoryURL string +} + +// startNodeInfoForTest builds and starts node-info on an available port. +// It registers cleanup and bounds the child's lifetime so stalled reads end. +func startNodeInfoForTest(t *testing.T, hostUUID string) *nodeInfoTestProcess { + t.Helper() + path := filepath.Join(t.TempDir(), "nvpair-node-info") + if runtime.GOOS == "windows" { + path += ".exe" + } + build := exec.Command("go", "build", "-o", path, ".") + output, err := build.CombinedOutput() + require.NoError(t, err, "build node-info: %s", output) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + _, portString, err := net.SplitHostPort(listener.Addr().String()) + require.NoError(t, err) + require.NoError(t, listener.Close()) + port, err := strconv.Atoi(portString) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + cmd := exec.CommandContext(ctx, path, "--node-id", hostUUID, "--port", portString) + stdin, err := cmd.StdinPipe() + require.NoError(t, err) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = stdin.Close() + assert.NoError(t, cmd.Wait()) + }) + return &nodeInfoTestProcess{ + stdin: stdin, + codec: jsonrpc.NewCodec(readWritePair{Reader: stdout, Writer: stdin}), + hostUUID: hostUUID, + port: port, + inventoryURL: "http://127.0.0.1:" + portString + "/v1/node-info", + } +} + +// nextSnapshot skips unrelated messages and decodes the next inventory update. +// Every returned snapshot must report the identity assigned to the child. +func (process *nodeInfoTestProcess) nextSnapshot(t *testing.T) noderec.NodeInfo { + t.Helper() + for { + msg, err := process.codec.Read() + require.NoError(t, err) + if msg.Method != noderec.NotifyInfoUpdated { + continue + } + var params noderec.LocalNodeInfoParams + require.NoError(t, json.Unmarshal(msg.Params, ¶ms)) + require.Equal(t, process.hostUUID, params.Info.HostUUID) + return params.Info + } +} + +// requestPeerMetadata sends metadata and waits for its matching acknowledgement, +// skipping periodic notifications that arrive while the request is in flight. +func (process *nodeInfoTestProcess) requestPeerMetadata(t *testing.T, metadata noderec.PeerMetadataParams) metadataAcknowledgement { + t.Helper() + id := json.RawMessage("1") + params, err := json.Marshal(metadata) + require.NoError(t, err) + request := jsonrpc.Message{JSONRPC: "2.0", ID: &id, Method: noderec.MethodSetPeerMetadata, Params: params} + require.NoError(t, json.NewEncoder(process.stdin).Encode(request)) + for { + msg, err := process.codec.Read() + require.NoError(t, err) + if msg.ID == nil || string(*msg.ID) != string(id) { + continue + } + require.Nil(t, msg.Error) + var acknowledgement metadataAcknowledgement + require.NoError(t, json.Unmarshal(msg.Result, &acknowledgement)) + return acknowledgement + } +} + +type readWritePair struct { + io.Reader + io.Writer +} + +type metadataAcknowledgement struct { + OK bool `json:"ok"` +} + +func TestInfoUpdates_PublishesStartupAndPeriodicSnapshots(t *testing.T) { + // Drive startup and one controlled tick to verify unsolicited publication + // without waiting for a real timer or making any HTTP requests. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + ticks := make(chan time.Time, 1) + ticks <- time.Now() + var output bytes.Buffer + var calls int + body := func() []byte { + calls++ + if calls == 2 { + cancel() + } + return buildResponse(nil, nil, 0, statsSnapshot{}, "owned-host", nil) + } + publishInfoUpdates(ctx, applog.NewNotifier(&output), body, ticks) + decoder := json.NewDecoder(&output) + for i := 0; i < 2; i++ { + var frame jsonrpc.Message + require.NoError(t, decoder.Decode(&frame), "snapshot %d", i) + var params noderec.LocalNodeInfoParams + require.NoError(t, json.Unmarshal(frame.Params, ¶ms)) + assert.Equal(t, noderec.NotifyInfoUpdated, frame.Method, "snapshot %d", i) + assert.Equal(t, "owned-host", params.Info.HostUUID, "snapshot %d", i) + assert.False(t, params.Info.TelemetryValid, "snapshot %d", i) + } + assert.Equal(t, 2, calls) + assert.False(t, decoder.More(), "only startup and one tick should publish") +} + +func TestPeerMetadata_ReplacesTheCompleteSnapshot(t *testing.T) { + // Apply successive registry snapshots through the stdin handler and verify + // acknowledgement and stored contents, including removal of obsolete fields. + var metadata peerMetadata + // Send one request and compare its acknowledged snapshot with stored state. + apply := func(params noderec.PeerMetadataParams) { + t.Helper() + body, err := json.Marshal(params) + require.NoError(t, err) + var output bytes.Buffer + handlePeerMetadata(applog.StdinMessage{ID: json.RawMessage(`1`), Method: noderec.MethodSetPeerMetadata, Params: body}, "owned-host", &metadata, applog.NewNotifier(&output)) + var response jsonrpc.Message + require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &response)) + var acknowledgement metadataAcknowledgement + require.NoError(t, json.Unmarshal(response.Result, &acknowledgement)) + require.Nil(t, response.Error) + require.True(t, acknowledgement.OK) + got, known := metadata.get() + require.True(t, known) + assert.Equal(t, params, got) + } + apply(noderec.PeerMetadataParams{ + HostUUID: "owned-host", ClusterUUID: "principal", + Peer: noderec.PeerMetadata{Name: "old-name", Addresses: []string{"192.0.2.1"}, Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: 14318, noderec.ServiceOllama: 11434}}, + }) + apply(noderec.PeerMetadataParams{ + HostUUID: "owned-host", + Peer: noderec.PeerMetadata{Name: "new-name", Addresses: []string{"192.0.2.2"}, Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: 14318}}, + }) +} + +func TestPeerMetadata_RejectsMalformedOrForeignIdentity(t *testing.T) { + // Invalid requests must return an invalid-params error and leave the worker + // without registry metadata that it could later serve as its own. + test := func(name, params string) { + t.Run(name, func(t *testing.T) { + var metadata peerMetadata + var output bytes.Buffer + handlePeerMetadata(applog.StdinMessage{ID: json.RawMessage(`2`), Method: noderec.MethodSetPeerMetadata, Params: json.RawMessage(params)}, "owned-host", &metadata, applog.NewNotifier(&output)) + var response jsonrpc.Message + require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &response)) + require.NotNil(t, response.Error) + assert.Equal(t, -32602, response.Error.Code) + _, known := metadata.get() + assert.False(t, known, "rejected metadata must not be stored") + }) + } + test("malformed object", `"invalid"`) + test("foreign host", `{"hostUuid":"someone-else","peer":{"name":"copied-name"}}`) +} diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index e7ce04c9a..17cd8f0d4 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -14,10 +14,11 @@ Concretely, the daemon: - **Enriches** each discovered node: GPU/CPU/memory inventory from its `ni` (node-info) port over plain HTTP, and the model list from its `em` (engine-manager) port (`GET /v1/models`) over cluster mTLS — a peer's model inventory is cluster data, so it is fetched only when this node holds a pin for the principal that peer advertises, while this node's own list is read over loopback in plaintext — the flat union, the per-engine breakdown (`modelsByEngine`), and the per-engine set of models loaded in memory (`loadedByEngine`), all carried onto the node's directory entry so remote cards can show loaded state. Each enrichment has a last-good cache so a transient fetch miss doesn't blank the node's card. Node-info enrichment can optionally be moved onto HTTPS (see the TLS flags below) — off by default and gated only on operator flags, never inferred per-node. - **Samples scheduling telemetry** from healthy nodes every two seconds with stable per-node jitter. Consecutive remote failures back off to 4, 8, 16, then - 30 seconds and reset on success; local loopback sampling stays at the healthy - cadence. It emits only host UUID, maximum GPU utilization, validity, and - source-reported sample age; it does not mutate the public directory for each - telemetry sample. + 30 seconds and reset on success. Local inventory and samples arrive every two + seconds from the owned node-info worker over private broker/worker IPC. + It emits only host UUID, maximum GPU utilization, validity, and + source-reported sample age. Remote telemetry reads do not mutate the public + directory; local IPC also updates changed hardware inventory. - **Annotates trust:** with `--cluster-dir`, a browsed peer whose `cluster-uuid=` this node holds a pin for is marked `trusted` (via `nvpair-shared/clustertrust`). The annotation is re-derived on `discovery:reload-trust`, which the broker sends when `nvpair-cluster-manager` reports a pin-set change, as well as when a peer's mDNS record changes. Both triggers are needed: a peer that was already advertising when this node joined never moves its record again, so the browse path alone would leave it annotated from before this node's own pins existed. A change emits one `discovery:node-updated`; a reload that finds nothing to change is silent. - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. - **Anti-flap:** a node has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. @@ -57,7 +58,7 @@ Bidirectional newline-delimited JSON-RPC 2.0. Stdio by default; `--ipc ` s `log/set-level` is also accepted. -Two notifications are accepted rather than requests, because the daemon has +These notifications are accepted rather than requests, because the daemon has nothing to answer and the caller must not wait: - `discovery:node-activity` `{hostUuid, msSince}` — a peer's engine returned @@ -66,6 +67,13 @@ nothing to answer and the caller must not wait: id-bearing call would cost it a pending entry and a blocked goroutine per report in exchange for an ack nobody reads. - `log/set-level` (above). +- `discovery:local-node-info` `{info}` — hardware received by the broker from its + own node-info worker. Scanner attaches it only to the matching local registry + record, updates inventory, and emits compact scheduling telemetry. It never + polls public node-info for local service data. A browsed UUID or loopback + address cannot enter this private path. Registration changes retain the owned + inventory; worker silence leaves scheduling samples aging rather than + generating fresh zero-usage samples. ### Notifications (daemon → caller) diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 778ebaded..7db6a843c 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -84,12 +84,12 @@ type daemon struct { // can never invite it back. clusterDir string - // selfMu serializes every mutation of this node's own directory entry - // (publishSelf / reloadIdentity / dropSelf) so a burst of service - // registrations and a concurrent network/identity change can't interleave - // into a torn self entry. The self key is written nowhere else — onBrowse - // ignores our own uuid — so this is the sole self-key writer lock. + // selfMu serializes registry publications, identity/address changes and owned + // hardware snapshots so they agree on this node's identity. Background model + // refreshes use dir.mu independently. selfMu sync.Mutex + // localInfo is supplied only by the owned worker IPC path, under selfMu. + localInfo *noderec.NodeInfo // lastInfo / lastModels cache each node's last successful enrichment (keyed by // hostUuid) — GPU/CPU/memory from node-info and the model list from @@ -133,8 +133,6 @@ type daemon struct { enrichHosts hostMemory // telemetryRetries owns one active telemetry attempt per node and suppresses // repeated remote attempts after failures at the same node-info target. - // Local results always finish successfully so loopback stays on the healthy - // cadence. telemetryRetries telemetryRetryGate // observedMu guards observed, this host's own addresses that remote peers @@ -887,10 +885,11 @@ func (d *daemon) publishSelfLocked() { Services: svcs, LastSeen: time.Now().Unix(), } - // Enrich over loopback, never the advertised LAN ip=: a first-run Windows - // firewall block on inbound to our own LAN address must not blank the local - // node's metrics/models (the "no metrics" failure mode). - d.enrichAt(&node, loopbackHost) + if d.localInfo != nil && d.localInfo.HostUUID == rec.HostUUID { + node.GPUs, node.CPU, node.Memory = d.localInfo.GPUs, d.localInfo.CPU, d.localInfo.Memory + } + // Models still come from the locally owned engine-manager over loopback. + d.enrichModelsCandidates(&node, []string{loopbackHost}) method := noderec.NotifyNodeUpdated if isNew := d.dir.upsert(node); isNew { @@ -899,6 +898,40 @@ func (d *daemon) publishSelfLocked() { d.emit(method, node) } +// acceptLocalNodeInfo applies an owned IPC snapshot to its matching local +// registry record. It updates changed inventory and emits maximum GPU usage +// for scheduling, including time spent processing the sample in its age. +func (d *daemon) acceptLocalNodeInfo(info noderec.NodeInfo) { + receivedAt := time.Now() + d.selfMu.Lock() + defer d.selfMu.Unlock() + self := d.reg.record().HostUUID + if self == "" || info.HostUUID != self { + return + } + info = info.Aged(0) + d.localInfo = &info + node, result := d.dir.applyLocalInfo(info) + switch result { + case localInfoMissing: + // Build identity and services from the registry, not the hardware snapshot. + d.publishSelfLocked() + case localInfoUpdated: + d.emit(noderec.NotifyNodeUpdated, node) + } + var utilization uint32 + for _, gpu := range info.GPUs { + if gpu.UtilizationPercent > utilization { + utilization = gpu.UtilizationPercent + } + } + info = info.Aged(time.Since(receivedAt)) + d.emitTelemetry(noderec.NodeTelemetry{ + HostUUID: self, GPUUtilizationPct: utilization, + TelemetryValid: info.TelemetryValid, MSSince: info.MSSince, + }) +} + // watchAddress re-derives the advertised ip= whenever the host's network changes // (interface or address add/remove — VPN/dock toggles, DHCP re-lease, sleep/wake // re-IP). Without it the ip= would be frozen at daemon startup and could point at @@ -949,10 +982,8 @@ const reachableInfoFreshness = 10 * time.Second // is to cover a whole burst of load rather than to require continuous traffic. const activityFreshness = 60 * time.Second -// loopbackHost is the address the daemon uses to reach THIS node's own services. -// Loopback is never firewalled, so enriching/probing self over it (rather than -// the advertised LAN ip=) makes the local node immune to a first-run Windows -// firewall block on inbound to its own LAN address. +// loopbackHost reaches the owned engine-manager for model inventory. Local +// node-info inventory and telemetry arrive over private worker IPC. const loopbackHost = "127.0.0.1" // reachable reports whether a threshold-missed node is still THERE — an mDNS @@ -992,13 +1023,13 @@ const loopbackHost = "127.0.0.1" // nothing to probe and evicts normally. func (d *daemon) reachable(n RawNode) bool { rec := noderec.ParseTXT(n.TXT) + if rec.HostUUID != "" && d.reg != nil && rec.HostUUID == d.reg.record().HostUUID { + return false // an mDNS advertisement cannot establish this host's local identity + } candidates := netpick.Candidates(n.TXT, n.Addresses) if len(candidates) == 0 { return false } - if rec.HostUUID != "" && d.reg != nil && rec.HostUUID == d.reg.record().HostUUID { - candidates = []string{loopbackHost} - } if rec.HostUUID != "" { d.infoMu.Lock() @@ -1137,24 +1168,13 @@ func (d *daemon) probeIdentity(ip string, port int) (NodeInfoResponse, bool) { // policy, and its engine-manager model list, which is cluster data and so is // fetched from a peer only over cluster mTLS (see fetchModels). Both are // independent (a node may advertise one without the other). Peers are dialed at -// each address they published until one answers for the expected identity; self -// is dialed over loopback (see enrichAt). +// each address they published until one answers for the expected identity. func (d *daemon) enrich(node *noderec.DirectoryNode) { hosts := node.CandidateIPs() d.enrichInfoCandidates(node, hosts) d.enrichModelsCandidates(node, hosts) } -// enrichAt is enrich against an explicit dial host. onBrowse enriches a peer at -// its advertised ip=; publishSelf enriches this node over loopbackHost so a -// first-run firewall block on our own LAN address can't blank the local card. -// The enrichment cache stays keyed by hostUuid, so the resilience -// (reuse-last-good on a transient miss) is identical for self and peers. -func (d *daemon) enrichAt(node *noderec.DirectoryNode, host string) { - d.enrichInfoCandidates(node, []string{host}) - d.enrichModelsCandidates(node, []string{host}) -} - // enrichInfoAt decorates a node that advertises node-info (ni) with its // GPU/CPU/memory inventory, fetched at the ni port on host. By default this is // PLAIN HTTP — node-info is plain per the transport policy, and the daemon must @@ -1322,7 +1342,7 @@ func (d *daemon) fetchNodeInfoWithin(ctx context.Context, ip string, port int) ( // pin for it — which models a machine holds is cluster data. A transient failure // reuses the last successful list rather than blanking it; a node advertising no em // has no model list (nothing is carried in the record). Self is dialed over -// loopback (see enrichAt), which stays plaintext, peers at their advertised address. +// loopback (see publishSelfLocked), which stays plaintext, peers at their advertised address. func (d *daemon) enrichModelsAt(node *noderec.DirectoryNode, host string) { d.enrichModelsCandidates(node, []string{host}) } @@ -1543,10 +1563,10 @@ func (d *daemon) dispatchTelemetry( if !ok { continue } - hosts := append([]string(nil), n.CandidateIPs()...) if n.HostUUID == self { - hosts = []string{loopbackHost} + continue // local telemetry arrives through discovery:local-node-info } + hosts := append([]string(nil), n.CandidateIPs()...) if len(hosts) == 0 { continue } @@ -1557,7 +1577,7 @@ func (d *daemon) dispatchTelemetry( } workers.Add(1) - go func(hostUUID string, hosts []string, port int, token uint64, backoff bool) { + go func(hostUUID string, hosts []string, port int, token uint64) { defer workers.Done() select { case sem <- struct{}{}: @@ -1567,11 +1587,11 @@ func (d *daemon) dispatchTelemetry( return } ok := d.refreshNodeTelemetryCandidates(ctx, hostUUID, hosts, port) - if !backoff || ctx.Err() != nil { + if ctx.Err() != nil { ok = true } d.telemetryRetries.finish(hostUUID, token, ok, time.Now()) - }(n.HostUUID, hosts, ni.Port, retryToken, n.HostUUID != self) + }(n.HostUUID, hosts, ni.Port, retryToken) } } @@ -1928,6 +1948,15 @@ func (d *daemon) activitySince(hostUUID string) (time.Duration, bool) { // filtered directory snapshot. func (d *daemon) handle(msg *Message) bool { switch msg.Method { + case noderec.MethodLocalNodeInfo: + var params noderec.LocalNodeInfoParams + if err := json.Unmarshal(msg.Params, ¶ms); err != nil { + slog.Warn("invalid local node-info snapshot", "err", err) + return true + } + d.acceptLocalNodeInfo(params.Info) + return true + case noderec.MethodRegister, noderec.MethodUpdateTXT: var p noderec.RegisterParams if err := json.Unmarshal(msg.Params, &p); err != nil { diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index 51f85480e..ddbf98eb3 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -4,6 +4,8 @@ package main import ( + "bytes" + "context" "encoding/json" "fmt" "net" @@ -13,6 +15,7 @@ import ( "os" "path/filepath" "strconv" + "sync/atomic" "testing" "time" @@ -123,44 +126,84 @@ func TestBrowse_DoesNotEvictLocalNode(t *testing.T) { } } -// TestPublishSelfEnrichesOverLoopback covers the "no metrics" failure mode: -// self enrichment must dial loopback, not the advertised LAN ip=, so a firewall -// block on inbound to our own LAN address can't blank the local card. The stub -// listens on loopback while the node advertises an (unreachable-in-test) LAN IP. -func TestPublishSelf_EnrichesOverLoopback(t *testing.T) { +func TestLocalNodeInfo_AttachesOwnedMeasurements(t *testing.T) { + // Verify that private worker samples populate the local registry record and + // scheduling telemetry before membership and after departure. Re-publishing + // local service registrations must retain that owned inventory. want := NodeInfoResponse{ - GPUs: []GPUInfo{{Name: "NVIDIA GeForce RTX 5090"}}, - CPU: &CPUInfo{Name: "AMD Ryzen 7 9800X3D", Cores: 8}, - Memory: &MemoryInfo{TotalBytes: 68719476736}, + HostUUID: "self-uuid", + GPUs: []GPUInfo{{Name: "NVIDIA GeForce RTX 5090"}}, + CPU: &CPUInfo{Name: "AMD Ryzen 7 9800X3D", Cores: 8}, + Memory: &MemoryInfo{TotalBytes: 68719476736}, + TelemetryValid: true, + MSSince: 137, } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - w.WriteHeader(http.StatusNotFound) - return - } - assert.NoError(t, json.NewEncoder(w).Encode(want)) + want.GPUs[0].UtilizationPercent = 61 + type localInfoParams struct { + Info NodeInfoResponse `json:"info"` + } + test := func(name string, leftCluster bool) { + t.Run(name, func(t *testing.T) { + d := newSelfTestDaemon("self-uuid", "192.0.2.17") + if leftCluster { + d.reg.setIdentity("self-uuid", "former-principal", selfAddrs("192.0.2.17")) + d.reg.setIdentity("self-uuid", "", selfAddrs("192.0.2.17")) + } + d.reg.register(noderec.RegisterParams{Service: noderec.ServiceNodeInfo, Port: 14318}) + params, err := json.Marshal(localInfoParams{Info: want}) + require.NoError(t, err) + var output bytes.Buffer + d.codec = NewCodec(&output) + require.True(t, d.handle(&Message{Method: "discovery:local-node-info", Params: params})) + node, ok := d.dir.get("self-uuid") + require.True(t, ok) + assert.Equal(t, noderec.IdentityLocal, node.IdentityVerification) + assert.Equal(t, "192.0.2.17", node.IP) + assert.Equal(t, want.GPUs, node.GPUs) + assert.Equal(t, want.CPU, node.CPU) + assert.Equal(t, want.Memory, node.Memory) + // A later registration must retain the owned hardware sample. + d.reg.register(noderec.RegisterParams{Service: noderec.ServiceOllama, Port: 11434}) + d.publishSelf() + node, _ = d.dir.get("self-uuid") + assert.Equal(t, want.GPUs, node.GPUs, "registration must retain local inventory") + var telemetry noderec.NodeTelemetry + decoder := json.NewDecoder(&output) + for decoder.More() { + var msg Message + require.NoError(t, decoder.Decode(&msg)) + if msg.Method == noderec.NotifyNodeTelemetry { + require.NoError(t, json.Unmarshal(msg.Params, &telemetry)) + } + } + assert.Equal(t, "self-uuid", telemetry.HostUUID) + assert.True(t, telemetry.TelemetryValid) + assert.Equal(t, uint32(61), telemetry.GPUUtilizationPct) + assert.GreaterOrEqual(t, telemetry.MSSince, int64(137)) + }) + } + test("before joining", false) + test("after leaving", true) +} + +func TestLocalNodeInfo_NeverPollsPublicInventory(t *testing.T) { + // Provide a public endpoint that would answer local reads, then verify that + // local publication and telemetry sampling issue no requests to it. + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte(`{"hostUuid":"self-uuid","GPUs":[{"name":"unverified hardware"}]}`)) })) - defer srv.Close() - _, portStr, err := net.SplitHostPort(srv.Listener.Addr().String()) - require.NoError(t, err, "split stub addr") - port, err := strconv.Atoi(portStr) + t.Cleanup(server.Close) + serverURL, err := url.Parse(server.URL) require.NoError(t, err) - - // The node advertises the LAN address (which no server answers on in the - // test); enrichment must still succeed via loopback, where the stub listens. - d := newSelfTestDaemon("self-uuid", "192.168.1.17") + port, err := strconv.Atoi(serverURL.Port()) + require.NoError(t, err) + d := newSelfTestDaemon("self-uuid", "192.0.2.17") d.reg.register(noderec.RegisterParams{Service: noderec.ServiceNodeInfo, Port: port}) d.publishSelf() - - n, ok := d.dir.get("self-uuid") - require.True(t, ok, "self missing from directory") - assert.Equal(t, "192.168.1.17", n.IP, "self display IP") - require.Len(t, n.GPUs, 1, "self GPUs") - assert.Equal(t, "NVIDIA GeForce RTX 5090", n.GPUs[0].Name, "self GPUs") - require.NotNil(t, n.CPU, "self CPU") - assert.Equal(t, "AMD Ryzen 7 9800X3D", n.CPU.Name, "self CPU") - require.NotNil(t, n.Memory, "self Memory") - assert.Equal(t, uint64(68719476736), n.Memory.TotalBytes, "self Memory") + d.refreshTelemetryOnce(context.Background()) + assert.Equal(t, int32(0), requests.Load()) } // TestPublishSelfLoopbackIPFallback covers the "IP unknown at startup" case: when @@ -299,39 +342,16 @@ func TestEnrich_FallsBackToASecondPublishedAddress(t *testing.T) { require.Equal(t, "GPU from reachable address", node.GPUs[0].Name) } -// TestReachableSelfDialsLoopback pins the uuid-keyed self probe: when the -// threshold-missed record is this host, reachable dials loopbackHost rather than -// the advertised LAN ip=. A hostname-keyed browse filter is not required — and -// would wrongly drop a peer that merely shares our instance name. -func TestReachableSelf_DialsLoopback(t *testing.T) { +func TestReachable_RejectsAdvertisementsClaimingTheLocalHost(t *testing.T) { + // Reject a discovery claim to our UUID even when a fresh cached sample + // would otherwise make that UUID reachable, without any network probe. const selfUUID = "self-uuid" - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - http.NotFound(w, r) - return - } - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: selfUUID})) - })) - defer srv.Close() - _, portStr, err := net.SplitHostPort(srv.Listener.Addr().String()) - require.NoError(t, err, "split stub addr") - port, err := strconv.Atoi(portStr) - require.NoError(t, err, "stub port") - d := newSelfTestDaemon(selfUUID, "192.0.2.1") - // Advertise an unroutable DOCUMENTATION address so a LAN dial cannot succeed; - // the stub only listens on loopback. + d.lastInfoAt[selfUUID] = time.Now() self := RawNode{ - ID: "myhost", - TXT: []string{"v=1", "uuid=" + selfUUID, "ip=192.0.2.1", fmt.Sprintf("ni=%d", port)}, - } - require.True(t, d.reachable(self), "self record must stay reachable via loopback when LAN ip= is unroutable") - - peer := RawNode{ - ID: "myhost", // same instance name as self — must not be treated as us - TXT: []string{"v=1", "uuid=peer-uuid", "ip=192.0.2.1", fmt.Sprintf("ni=%d", port)}, + TXT: []string{"uuid=" + selfUUID, "ip=192.0.2.1"}, } - assert.False(t, d.reachable(peer), "a same-hostname peer advertising an unroutable LAN ip must not inherit the self loopback dial") + assert.False(t, d.reachable(self), "an advertisement claiming our UUID must be rejected despite a fresh cached sample") } func TestPublishSelf_LocalRecordHasLocalVerificationAndDiscoveryRecordID(t *testing.T) { diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index aafe792cb..613e0fe6a 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "fmt" "log/slog" + "slices" "sort" "strconv" "sync" @@ -79,9 +80,9 @@ func nodeDiscoveryRecordID(raw RawNode) string { return fmt.Sprintf("observation:%x", sha256.Sum256([]byte(fmt.Sprintf("%q", fields)))) } -// directory is the daemon's view of all LAN nodes (this node included), keyed by -// hostUuid. Mutations come from the browse-event goroutine; reads come from -// discovery:get-nodes handlers, so it's an RWMutex map. +// directory stores this node and discovered peers by hostUuid. Browse events, +// IPC handlers and background refreshes access it concurrently, so reads and +// mutations share mu. type directory struct { mu sync.RWMutex nodes map[string]noderec.DirectoryNode @@ -240,6 +241,41 @@ func (d *directory) snapshot(filter noderec.ServiceKey) []noderec.DirectoryNode return out } +type localInfoResult uint8 + +const ( + localInfoMissing localInfoResult = iota + localInfoUnchanged + localInfoUpdated +) + +// applyLocalInfo changes only hardware under one directory lock. Background +// model refreshes do not take selfMu, so separate get/upsert calls could overwrite +// their newer model inventory with the hardware handler's earlier copy. +func (d *directory) applyLocalInfo(info noderec.NodeInfo) (noderec.DirectoryNode, localInfoResult) { + d.mu.Lock() + defer d.mu.Unlock() + node, present := d.nodes[info.HostUUID] + if !present { + return node, localInfoMissing + } + if slices.Equal(node.GPUs, info.GPUs) && equalOptional(node.CPU, info.CPU) && equalOptional(node.Memory, info.Memory) { + return node, localInfoUnchanged + } + node.GPUs, node.CPU, node.Memory = info.GPUs, info.CPU, info.Memory + node.LastSeen = time.Now().Unix() + d.nodes[info.HostUUID] = node + return node, localInfoUpdated +} + +// equalOptional distinguishes an absent value from a present zero value. +func equalOptional[T comparable](a, b *T) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} + // applyModels atomically reconciles one node's model inventory from a completed // /v1/models fetch, guarded against a node that was removed or re-addressed // while the fetch was in flight. It is the periodic refresh loop's write path diff --git a/services/nvpair-node-scanner/directory_test.go b/services/nvpair-node-scanner/directory_test.go index 92276e1fe..777e92508 100644 --- a/services/nvpair-node-scanner/directory_test.go +++ b/services/nvpair-node-scanner/directory_test.go @@ -12,6 +12,29 @@ import ( "nvpair-shared/noderec" ) +func TestApplyLocalInfo_UpdatesOnlyHardware(t *testing.T) { + dir := newDirectory() + node := noderec.DirectoryNode{ + HostUUID: "owned-host", DiscoveryRecordID: "local:owned-host", + IdentityVerification: noderec.IdentityLocal, Trusted: true, + Name: "workstation", IP: "192.0.2.20", ClusterUUID: "principal", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14322}}, + Models: []string{"current-model"}, LastSeen: 1, + } + dir.upsert(node) + info := noderec.NodeInfo{ + HostUUID: "owned-host", GPUs: []GPUInfo{{Name: "GPU", UtilizationPercent: 61}}, + CPU: &CPUInfo{Name: "CPU"}, Memory: &MemoryInfo{TotalBytes: 1024}, + } + got, result := dir.applyLocalInfo(info) + require.Equal(t, localInfoUpdated, result) + want := node + want.GPUs, want.CPU, want.Memory = info.GPUs, info.CPU, info.Memory + // Ignore the refresh timestamp when checking preservation of other fields. + want.LastSeen = got.LastSeen + assert.Equal(t, want, got) +} + func TestToDirectoryNode_ProjectsIdentityServicesAndTrust(t *testing.T) { raw := RawNode{ ID: "hostA", diff --git a/services/nvpair-node-scanner/scanner.go b/services/nvpair-node-scanner/scanner.go index 485a0275a..8450ed379 100644 --- a/services/nvpair-node-scanner/scanner.go +++ b/services/nvpair-node-scanner/scanner.go @@ -30,27 +30,9 @@ type ( ) // NodeInfoResponse is the /v1/node-info body the daemon fetches to enrich a -// node (see daemon.fetchNodeInfo). -type NodeInfoResponse struct { - GPUs []GPUInfo `json:"GPUs"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` - // HostUUID is the identity the node itself reports (nvpair-node-info's - // hostUuid). It is not enrichment data — it is what lets the liveness probe - // tell "this node still answers" from "some other node now answers at this - // address", which is the difference between a live peer and the ghost a - // wiped-and-reinstalled node leaves behind. Empty when the peer predates the - // field, in which case the probe falls back to its TCP-only behaviour. - HostUUID string `json:"hostUuid,omitempty"` - // ClusterUUID is the node's own report of the cluster principal it holds, - // empty when it belongs to no cluster. A pointer so absent is distinguishable - // from empty: a node too old to report the field says nothing about its - // membership, while a node reporting "" is asserting it has none. Reading - // absent as unclustered would mark a clustered peer invitable. - ClusterUUID *string `json:"clusterUuid"` -} +// node. Share the hardware contract with the local IPC path to prevent drift. +// A nil ClusterUUID is unknown membership, while a pointer to "" is unclustered. +type NodeInfoResponse = noderec.NodeInfo type ReadyParams struct { Version string `json:"version"` diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 40f72466e..b8f7f3fcd 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -66,6 +66,20 @@ There is **one supervisor per process, not per engine**. `nvpair-proxy` hosts a Bidirectional newline-delimited JSON-RPC 2.0 — same conventions as every other NVPAIR subprocess. By default the broker reads stdin and writes stdout; with `--ipc ` it dials a Unix domain socket or Windows named pipe instead. The parent UI process owns the endpoint in both cases: it either spawns the broker with piped stdio, or sets up the pipe / socket beforehand and points the broker at it via `--ipc`. The broker always talks to exactly one peer. +The owned node-info worker emits `nodeinfo:info-updated {info}` at startup and +every two seconds. The broker retains the latest snapshot and sends it to scanner +as `discovery:local-node-info {info}`, including after scanner restarts. Queue and +cache time increase the reported sample age. The relay queue is bounded, and +worker silence cannot make an old sample freshly idle. A snapshot naming a host +other than this broker's host is rejected. + +Local scanner directory events marked `identityVerification:local` supply +`nodeinfo:set-peer-metadata {hostUuid, clusterUuid, peer}` to node-info. The +complete registry metadata replaces the prior snapshot when names, addresses, +membership, or registrations change, and is replayed after node-info restarts. +Remote discovery claims cannot populate this metadata. These links are internal +worker IPC; desktop and TUI continue using public HTTP for hardware displays. + ## CLI Flags | Flag | Default | Description | diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index b62d76529..5c0579864 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -208,6 +208,12 @@ type Broker struct { lmstudioReadyMu sync.Mutex store *discoveryStore telemetry *telemetryCache + localInfoMu sync.Mutex + localInfo *localInfoObservation + peerMetadataMu sync.Mutex + localPeerMetadata *noderec.PeerMetadataParams + peerMetadataSending bool // guarded by peerMetadataMu + peerMetadataPending bool // guarded by peerMetadataMu // relayDir is the discovery directory, fed by the promoted daemon's // discovery:node-* stream. It runs alongside the legacy store during the // migration; the client-facing get-nodes/subscribe path moves onto it at the @@ -613,6 +619,7 @@ func (b *Broker) getClusterMgr() *clusterManagerProcess { // and returns it as a supervisedHandle for the supervisor to watch. A // spawn failure returns the error without storing a handle, leaving the // previous (dead) handle in place until the next successful (re)start. +// spawnScanner also replays cached local inventory and service registrations. func (b *Broker) spawnScanner() (supervisedHandle, error) { sp, err := startScanner( b.scannerPath, @@ -622,12 +629,14 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { b.failWorkloadsForNode, func(telemetry noderec.NodeTelemetry) { b.ingestTelemetry(sourceScanner, telemetry) }, func(hostUUID string) { b.removeTelemetry(sourceScanner, hostUUID) }, + b.updateLocalPeerMetadata, b.clusterDirArgs()..., ) if err != nil { return nil, err } b.setScanner(sp) + b.replayLocalInfo(sp) // Replay cached registrations to the (re)started daemon so it re-advertises // this node's services without any worker reconnect logic (epoch replay). for _, p := range b.regCache.Snapshot() { @@ -637,21 +646,13 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { return sp, nil } +// spawnNodeInfo starts the owned inventory worker, restores identity and peer +// metadata, and registers its service port with scanner. func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { - // Deliberately NOT passed b.clusterDirArgs(): node-info stays plain HTTP even - // when clustered, and is the ONE documented exception to "the inter-node - // cluster data plane is always mTLS" (errors and workload-manager admit only - // pinned members, in either direction). Its GPU/CPU/memory inventory is the - // lowest-sensitivity inter-node surface and is read directly by consumers that - // hold no cluster identity — the desktop app's two-second telemetry poll and - // baseline LAN discovery — so gating it would remove node telemetry from the UI. - // The accepted risk and its compensating controls are recorded under - // "Documented Risk Acceptances" in desktop/SECURITY.md. - // - // node-info still implements the cluster-gated mode (pass it --cluster-dir and - // it admits only pinned peers). It is kept, unenabled, on purpose: it is the - // swap point if the desktop telemetry read is ever solved another way. Enabling - // it here without solving that first WILL blank the UI's node cards. + // Keep public HTTP available for desktop and TUI hardware displays. The + // current --cluster-dir mode gates that endpoint while clustered, so do not + // enable it here. Local service measurements use the owned worker's stdio, + // independent of whether this host belongs to a cluster. // // Do pass --node-id: node-info reports the broker's already-resolved UUID so // /v1/node-info agrees with the identity the fleet keys on, even on a custom @@ -666,6 +667,7 @@ func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { // /v1/node-info but holds no cluster dir to read it from, so this push is the // only source. It runs on every spawn, which also covers a supervised restart. b.pushClusterIdentityToNodeInfo() + b.pushPeerMetadata() // Register node-info's service so the daemon advertises ni= on _nvpair-node. // node-info binds the fixed :14318 (force_ports is inert), so the broker // knows its port. Idempotent across restarts. @@ -1797,12 +1799,14 @@ func (b *Broker) forwardSchedulerNotification(method string, params json.RawMess slog.Debug("ignoring scheduler notification", "method", method) } -// forwardNodeInfoNotification is the hook startNodeInfo invokes on node-info's -// reader goroutine for every notification it emits. Its observed local addresses -// go to the scanner, which owns what this node advertises: node-info learns which -// addresses peers actually reach us on, and that is the only direct evidence -// address selection can have about reachability from somewhere else. +// forwardNodeInfoNotification routes owned hardware snapshots and observed +// addresses to scanner from node-info's stdout reader. Hardware supplies local +// service measurements; observed addresses help rank this host's interfaces. func (b *Broker) forwardNodeInfoNotification(method string, params json.RawMessage) { + if method == noderec.NotifyInfoUpdated { + b.acceptLocalInfo(params) + return + } if method != noderec.NotifyObservedAddresses { slog.Debug("ignoring node-info notification", "method", method) return diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 48048a2e5..67db946d9 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -354,6 +354,8 @@ type scannerProcess struct { onNodeLost func(uuid, name string) onTelemetry func(noderec.NodeTelemetry) onTelemetryRemoved func(hostUUID string) + onLocalNode func(noderec.DirectoryNode) + localInfo chan localInfoObservation // activity queues peer liveness reports for the scanner. It exists so the // proxy-reader goroutine that receives them never waits on the scanner's // pipe: reports arrive while inference is streaming, and a wedged scanner @@ -400,6 +402,7 @@ func startScanner( onNodeLost func(uuid, name string), onTelemetry func(noderec.NodeTelemetry), onTelemetryRemoved func(hostUUID string), + onLocalNode func(noderec.DirectoryNode), extraArgs ...string, ) (*scannerProcess, error) { cmd := exec.Command(binaryPath, append([]string{"--log-level", logLevel}, extraArgs...)...) @@ -430,6 +433,8 @@ func startScanner( onNodeLost: onNodeLost, onTelemetry: onTelemetry, onTelemetryRemoved: onTelemetryRemoved, + onLocalNode: onLocalNode, + localInfo: make(chan localInfoObservation, 1), activity: make(chan nodeActivityReport, activityQueueDepth), done: make(chan struct{}), } @@ -530,8 +535,17 @@ func (s *scannerProcess) reportNodeActivity(hostUUID string, observedAt time.Tim } } -// drainNodeActivity forwards queued liveness reports to the daemon until the -// scanner exits. +// reportLocalInfo queues an owned-worker snapshot without blocking the caller. +// A full queue drops the new sample, allowing existing measurements to age. +func (s *scannerProcess) reportLocalInfo(observation localInfoObservation) { + select { + case s.localInfo <- observation: + default: + } +} + +// drainNodeActivity forwards queued liveness reports and owned inventory to the +// daemon until the scanner exits. // // They go as notifications, not requests. The daemon has nothing to say back — // it records the report and returns — so an id-bearing call would buy a pending @@ -541,6 +555,11 @@ func (s *scannerProcess) drainNodeActivity() { select { case <-s.done: return + case observation := <-s.localInfo: + info := observation.info.Aged(time.Since(observation.receivedAt)) + if err := s.peer.Notify(noderec.MethodLocalNodeInfo, noderec.LocalNodeInfoParams{Info: info}); err != nil { + slog.Debug("relay local node-info to daemon failed", "err", err) + } case report := <-s.activity: // Aged at send time, not enqueue time, so any wait in the queue is // included: the daemon measures freshness against this number. @@ -581,6 +600,9 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { slog.Warn("daemon emitted invalid node event", "method", method, "err", err) return } + if method != noderec.NotifyNodeRemoved && s.onLocalNode != nil { + s.onLocalNode(ev.Node) + } // Fold into the relay directory (internal subscribers: proxies, mesh // services) and the client-facing store (discovery:get-nodes / // nodes-changed, merging manual nodes), carrying the trusted annotation. diff --git a/services/nvpair-ui-broker/nodeinfo.go b/services/nvpair-ui-broker/nodeinfo.go index af890547f..b0c1f6af0 100644 --- a/services/nvpair-ui-broker/nodeinfo.go +++ b/services/nvpair-ui-broker/nodeinfo.go @@ -10,23 +10,24 @@ import ( "fmt" "io" "log/slog" + "maps" "os" "os/exec" + "slices" "sync" + "time" + + "nvpair-shared/jsonrpc" + "nvpair-shared/noderec" ) -// maxNodeInfoLine caps one stdout frame from node-info. Its frames are a handful -// of addresses or a control-request response, so this is generous; the cap only -// exists so a wedged child cannot grow the broker's buffer without bound. +// maxNodeInfoLine caps inventory snapshots, observed addresses and control +// responses so a wedged child cannot grow the broker's buffer without bound. const maxNodeInfoLine = 64 * 1024 // nodeInfoProcess is the broker's handle on its child nvpair-node-info. -// Unlike the scanner, node-info is a *server* (HTTP node-info endpoint + -// mDNS advertiser), not an event source: the broker doesn't consume its -// stdout into the discovery store. The broker spawns it so the broker's -// own host advertises -// its GPU / CPU / memory inventory on the network, where the scanner -// (also spawned by the broker) and remote peers can then discover it. +// It serves public hardware inventory and publishes owned hardware snapshots +// over stdout. The broker relays those snapshots to its local scanner. // // The handle owns the OS process, the stdin pipe used to signal clean // shutdown (via EOF) and to forward runtime control frames @@ -34,8 +35,8 @@ const maxNodeInfoLine = 64 * 1024 type nodeInfoProcess struct { cmd *exec.Cmd stdin io.WriteCloser - // stdinMu serializes log-level forwarding writes against the Close() - // in Stop() (this handle stays notify-only, not on jsonrpc.Peer). + // stdinMu keeps control frames from interleaving on the pipe. Stop closes + // stdin without this lock so it can interrupt a blocked write. stdinMu sync.Mutex done chan struct{} } @@ -58,6 +59,131 @@ func (n *nodeInfoProcess) SetClusterIdentity(clusterUUID string) error { return writeClusterIdentityFrame(&n.stdinMu, n.stdin, clusterUUID) } +type localInfoObservation struct { + info noderec.NodeInfo + receivedAt time.Time +} + +// acceptLocalInfo caches an owned-worker snapshot matching this host and queues +// it for a running scanner. Receipt time is retained for age-preserving replay. +func (b *Broker) acceptLocalInfo(params json.RawMessage) { + receivedAt := time.Now() + var snapshot noderec.LocalNodeInfoParams + if err := json.Unmarshal(params, &snapshot); err != nil { + slog.Warn("bad owned node-info snapshot", "err", err) + return + } + if b.nodeID == "" || snapshot.Info.HostUUID != b.nodeID { + slog.Warn("owned node-info snapshot has an unexpected host identity") + return + } + observation := localInfoObservation{info: snapshot.Info, receivedAt: receivedAt} + b.localInfoMu.Lock() + b.localInfo = &observation + b.localInfoMu.Unlock() + if scanner := b.getScanner(); scanner != nil { + scanner.reportLocalInfo(observation) + } +} + +// replayLocalInfo queues the cached snapshot for a restarted scanner, retaining +// its original receipt time so the relay includes time spent in the cache. +func (b *Broker) replayLocalInfo(scanner *scannerProcess) { + b.localInfoMu.Lock() + defer b.localInfoMu.Unlock() + if b.localInfo != nil { + scanner.reportLocalInfo(*b.localInfo) + } +} + +// updateLocalPeerMetadata projects this host's owned registry record into peer +// metadata, caching it for restart replay and forwarding changes to node-info. +func (b *Broker) updateLocalPeerMetadata(node noderec.DirectoryNode) { + if node.IdentityVerification != noderec.IdentityLocal || node.HostUUID != b.nodeID { + return + } + // Copy mutable fields so the writer can use this snapshot after releasing + // the cache lock, even if scanner supplies another registry update. + services := make(map[noderec.ServiceKey]int, len(node.Services)) + for key, status := range node.Services { + services[key] = status.Port + } + metadata := noderec.PeerMetadataParams{ + HostUUID: node.HostUUID, ClusterUUID: node.ClusterUUID, + Peer: noderec.PeerMetadata{Name: node.Name, Addresses: append([]string(nil), node.CandidateIPs()...), Services: services}, + } + b.peerMetadataMu.Lock() + previous := b.localPeerMetadata + changed := previous == nil || previous.HostUUID != metadata.HostUUID || + previous.ClusterUUID != metadata.ClusterUUID || previous.Peer.Name != metadata.Peer.Name || + !slices.Equal(previous.Peer.Addresses, metadata.Peer.Addresses) || + !maps.Equal(previous.Peer.Services, metadata.Peer.Services) + b.localPeerMetadata = &metadata + b.peerMetadataMu.Unlock() + if changed { + b.pushPeerMetadata() + } +} + +// pushPeerMetadata queues the latest registry snapshot. One sender handles all +// updates, with at most one pending update while the child's stdin is blocked. +func (b *Broker) pushPeerMetadata() { + if b.getNodeInfo() == nil { + return + } + b.peerMetadataMu.Lock() + b.peerMetadataPending = true + if b.peerMetadataSending { + b.peerMetadataMu.Unlock() + return + } + b.peerMetadataSending = true + b.peerMetadataMu.Unlock() + go b.drainPeerMetadata() +} + +// drainPeerMetadata exits when caught up. Restart replay queues another send; +// closing a stopped worker's stdin releases any write still using that worker. +func (b *Broker) drainPeerMetadata() { + for { + b.peerMetadataMu.Lock() + if !b.peerMetadataPending { + b.peerMetadataSending = false + b.peerMetadataMu.Unlock() + return + } + b.peerMetadataPending = false + b.peerMetadataMu.Unlock() + b.sendPeerMetadata() + } +} + +// sendPeerMetadata reads the cache after taking the current worker's pipe lock, +// then releases the cache lock before I/O so registry updates never wait for it. +func (b *Broker) sendPeerMetadata() { + process := b.getNodeInfo() + if process == nil { + return + } + process.stdinMu.Lock() + defer process.stdinMu.Unlock() + b.peerMetadataMu.Lock() + metadata := b.localPeerMetadata + b.peerMetadataMu.Unlock() + if metadata == nil { + return + } + params, err := json.Marshal(metadata) + if err != nil { + slog.Warn("failed to encode local peer metadata", "err", err) + return + } + frame := jsonrpc.Message{JSONRPC: "2.0", Method: noderec.MethodSetPeerMetadata, Params: params} + if err := json.NewEncoder(process.stdin).Encode(frame); err != nil { + slog.Warn("failed to push local peer metadata", "err", err) + } +} + // Done implements supervisedHandle: the returned channel closes once the // node-info process has exited (cmd.Wait returned). func (n *nodeInfoProcess) Done() <-chan struct{} { return n.done } diff --git a/services/nvpair-ui-broker/nodeinfo_test.go b/services/nvpair-ui-broker/nodeinfo_test.go new file mode 100644 index 000000000..0acadb2f0 --- /dev/null +++ b/services/nvpair-ui-broker/nodeinfo_test.go @@ -0,0 +1,330 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "nvpair-shared/noderec" +) + +func TestLocalNodeInfo_RelaysOwnedWorkerSamples(t *testing.T) { + // Follow an owned worker snapshot through the broker's notification handler + // and scanner relay. Hardware values, identity, validity, and sample age + // must survive that handoff. + b := &Broker{nodeID: "owned-host"} + sp, messages := localInfoScanner(t) + b.setScanner(sp) + b.forwardNodeInfoNotification(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","GPUs":[{"name":"GPU","utilization_percent":61}],"telemetryValid":true,"msSince":137}}`)) + info := readLocalInfo(t, messages) + assert.Equal(t, "owned-host", info.HostUUID) + assert.True(t, info.TelemetryValid) + assert.GreaterOrEqual(t, info.MSSince, int64(137)) + require.Len(t, info.GPUs, 1) + assert.Equal(t, uint32(61), info.GPUs[0].UtilizationPercent) +} + +func TestLocalNodeInfo_ReplayPreservesSampleAge(t *testing.T) { + // Simulate a snapshot waiting in the broker cache while scanner is absent. + // Replaying it must include that wait in its age so an old measurement + // cannot appear freshly sampled after scanner restarts. + b := &Broker{nodeID: "owned-host"} + b.forwardNodeInfoNotification(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","telemetryValid":true,"msSince":137}}`)) + require.NotNil(t, b.localInfo) + b.localInfo.receivedAt = time.Now().Add(-3 * time.Second) + sp, messages := localInfoScanner(t) + b.replayLocalInfo(sp) + info := readLocalInfo(t, messages) + assert.GreaterOrEqual(t, info.MSSince, int64(3137)) + assert.LessOrEqual(t, info.MSSince, int64(6000)) +} + +func TestLocalNodeInfo_RejectsForeignWorkerIdentity(t *testing.T) { + // A snapshot naming another host must not enter the locally owned sample + // cache, even when it reports valid telemetry. + b := &Broker{nodeID: "owned-host"} + b.forwardNodeInfoNotification(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"someone-else","telemetryValid":true}}`)) + assert.Nil(t, b.localInfo) +} + +// localInfoScanner connects the broker's production relay to an in-memory +// scanner stand-in. It returns the relay handle and messages received at the +// scanner end, and registers cleanup to stop the relay and close both ends. +func localInfoScanner(t *testing.T) (*scannerProcess, <-chan *Message) { + t.Helper() + client, server := net.Pipe() + done := make(chan struct{}) + t.Cleanup(func() { + close(done) + _ = client.Close() + _ = server.Close() + }) + sp := &scannerProcess{ + peer: NewPeer(NewCodec(client)), done: done, + localInfo: make(chan localInfoObservation, 1), + } + go sp.drainNodeActivity() + messages := make(chan *Message, 4) + go func() { + defer close(messages) + codec := NewCodec(server) + for { + msg, err := codec.Read() + if err != nil { + return + } + messages <- msg + } + }() + return sp, messages +} + +// readLocalInfo waits up to two seconds for a local inventory notification, +// checks its method, and decodes the hardware snapshot for the test to assert on. +func readLocalInfo(t *testing.T, messages <-chan *Message) noderec.NodeInfo { + t.Helper() + select { + case msg := <-messages: + require.NotNil(t, msg) + require.Equal(t, noderec.MethodLocalNodeInfo, msg.Method) + var params noderec.LocalNodeInfoParams + require.NoError(t, json.Unmarshal(msg.Params, ¶ms)) + return params.Info + case <-time.After(2 * time.Second): + require.FailNow(t, "owned snapshot did not reach the scanner") + return noderec.NodeInfo{} + } +} + +type nodeInfoFrameWriter struct { + frames chan []byte +} + +func (w *nodeInfoFrameWriter) Write(data []byte) (int, error) { + w.frames <- append([]byte(nil), data...) + return len(data), nil +} + +func (w *nodeInfoFrameWriter) Close() error { return nil } + +func TestPeerMetadata_RelaysAndReplaysOnlyTheOwnedRegistry(t *testing.T) { + // Only the owned local registry may supply node-info's peer metadata. + // Verify that unverified discoveries are ignored, cached metadata is replayed + // when node-info becomes available, and later snapshots withdraw membership + // and removed services instead of retaining stale fields. + b := &Broker{nodeID: "owned-host"} + node := noderec.DirectoryNode{ + HostUUID: "owned-host", IdentityVerification: noderec.IdentityUnverified, + Name: "copied-name", IP: "192.0.2.99", + } + b.updateLocalPeerMetadata(node) + require.Nil(t, b.localPeerMetadata, "unverified discovery must not supply local peer metadata") + node.IdentityVerification = noderec.IdentityLocal + node.Name, node.IP, node.ClusterUUID = "workstation", "192.0.2.20", "principal" + node.IPs = []string{"192.0.2.20", "192.0.2.21"} + node.Services = map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: 14318}, noderec.ServiceOllama: {Port: 11434}} + b.updateLocalPeerMetadata(node) + writer := &nodeInfoFrameWriter{frames: make(chan []byte, 4)} + b.setNodeInfo(&nodeInfoProcess{stdin: writer}) + b.pushPeerMetadata() + // Check one frame sent to node-info against the complete expected metadata. + // Reuse this for the initial replay and the later withdrawal of stale fields. + check := func(want noderec.PeerMetadataParams) { + t.Helper() + select { + case data := <-writer.frames: + var msg Message + require.NoError(t, json.Unmarshal(data, &msg)) + var got noderec.PeerMetadataParams + require.NoError(t, json.Unmarshal(msg.Params, &got)) + assert.Equal(t, noderec.MethodSetPeerMetadata, msg.Method) + assert.Equal(t, want, got) + case <-time.After(2 * time.Second): + require.FailNow(t, "local metadata did not reach node-info") + } + } + want := noderec.PeerMetadataParams{HostUUID: "owned-host", ClusterUUID: "principal", Peer: noderec.PeerMetadata{Name: node.Name, Addresses: node.IPs, Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: 14318, noderec.ServiceOllama: 11434}}} + check(want) + delete(node.Services, noderec.ServiceOllama) + node.ClusterUUID = "" + b.updateLocalPeerMetadata(node) + want.ClusterUUID = "" + delete(want.Peer.Services, noderec.ServiceOllama) + check(want) +} + +type blockedMetadataWriter struct { + net.Conn + started chan struct{} + once sync.Once +} + +func (w *blockedMetadataWriter) Write(data []byte) (int, error) { + w.once.Do(func() { close(w.started) }) + return w.Conn.Write(data) +} + +func TestPeerMetadata_UpdatesDoNotWaitForBlockedWrites(t *testing.T) { + b := &Broker{nodeID: "owned-host"} + node := noderec.DirectoryNode{ + HostUUID: "owned-host", IdentityVerification: noderec.IdentityLocal, + Name: "initial-name", + } + b.updateLocalPeerMetadata(node) + // Withhold reads until the cache update returns, simulating a worker that + // has stopped consuming its stdin. + client, server := net.Pipe() + t.Cleanup(func() { + _ = client.Close() + _ = server.Close() + }) + writer := &blockedMetadataWriter{Conn: client, started: make(chan struct{})} + b.setNodeInfo(&nodeInfoProcess{stdin: writer}) + go b.pushPeerMetadata() + select { + case <-writer.started: + case <-time.After(2 * time.Second): + require.FailNow(t, "metadata write did not start") + } + node.Name = "updated-name" + updated := make(chan struct{}) + go func() { + b.updateLocalPeerMetadata(node) + close(updated) + }() + select { + case <-updated: + case <-time.After(2 * time.Second): + require.FailNow(t, "a blocked node-info write stalled the scanner notification handler") + } + // Reading releases the blocked write; both snapshots must arrive in order. + require.NoError(t, server.SetReadDeadline(time.Now().Add(2*time.Second))) + codec := NewCodec(server) + checkName := func(want string) { + t.Helper() + msg, err := codec.Read() + require.NoError(t, err) + var params noderec.PeerMetadataParams + require.NoError(t, json.Unmarshal(msg.Params, ¶ms)) + assert.Equal(t, want, params.Peer.Name) + } + checkName("initial-name") + checkName("updated-name") +} + +type heldMetadataWriter struct { + frames chan []byte + started chan struct{} + release chan struct{} + once sync.Once + releaseOnce sync.Once +} + +func (w *heldMetadataWriter) Write(data []byte) (int, error) { + w.once.Do(func() { + close(w.started) + <-w.release + }) + w.frames <- append([]byte(nil), data...) + return len(data), nil +} + +func (w *heldMetadataWriter) Close() error { + w.releaseOnce.Do(func() { close(w.release) }) + return nil +} + +func waitForMetadataSender(t *testing.T, b *Broker) { + t.Helper() + // Join the asynchronous sender before inspecting all frames it produced. + require.Eventually(t, func() bool { + b.peerMetadataMu.Lock() + defer b.peerMetadataMu.Unlock() + return !b.peerMetadataSending + }, 2*time.Second, time.Millisecond) +} + +func TestPeerMetadata_CoalescesUpdatesWhileWorkerIsBlocked(t *testing.T) { + // A stopped reader must retain only the latest complete pending registry + // snapshot, rather than accumulating one sender and write per update. + const updates = 20 + b := &Broker{nodeID: "owned-host"} + writer := &heldMetadataWriter{ + frames: make(chan []byte, updates+2), started: make(chan struct{}), release: make(chan struct{}), + } + t.Cleanup(func() { require.NoError(t, writer.Close()) }) + b.setNodeInfo(&nodeInfoProcess{stdin: writer}) + node := noderec.DirectoryNode{ + HostUUID: "owned-host", IdentityVerification: noderec.IdentityLocal, Name: "initial", + IP: "192.0.2.20", ClusterUUID: "principal", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceNodeInfo: {Port: 14318}, noderec.ServiceOllama: {Port: 11434}, + }, + } + b.updateLocalPeerMetadata(node) + select { + case <-writer.started: + case <-time.After(2 * time.Second): + require.FailNow(t, "metadata write did not start") + } + for i := 1; i <= updates; i++ { + node.Name = fmt.Sprintf("updated-%d", i) + b.updateLocalPeerMetadata(node) + } + node.IP = "192.0.2.21" + node.ClusterUUID = "" + delete(node.Services, noderec.ServiceOllama) + b.updateLocalPeerMetadata(node) + + require.NoError(t, writer.Close()) + waitForMetadataSender(t, b) + + require.Len(t, writer.frames, 2, "only the in-flight and latest pending snapshots should be written") + <-writer.frames // The in-flight snapshot was already encoded before updates. + var frame Message + require.NoError(t, json.Unmarshal(<-writer.frames, &frame)) + assert.Equal(t, noderec.MethodSetPeerMetadata, frame.Method) + var got noderec.PeerMetadataParams + require.NoError(t, json.Unmarshal(frame.Params, &got)) + assert.Equal(t, noderec.PeerMetadataParams{ + HostUUID: "owned-host", + Peer: noderec.PeerMetadata{ + Name: "updated-20", Addresses: []string{"192.0.2.21"}, + Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: 14318}, + }, + }, got) +} + +func TestPeerMetadata_ClosingWorkerPipeReleasesSender(t *testing.T) { + // Shutdown closes stdin without taking its write lock, so a blocked metadata + // write and its pending update must both finish rather than leave a sender. + b := &Broker{nodeID: "owned-host"} + client, server := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, client.Close()) + assert.NoError(t, server.Close()) + }) + writer := &blockedMetadataWriter{Conn: client, started: make(chan struct{})} + b.setNodeInfo(&nodeInfoProcess{stdin: writer}) + node := noderec.DirectoryNode{HostUUID: "owned-host", IdentityVerification: noderec.IdentityLocal, Name: "initial"} + b.updateLocalPeerMetadata(node) + select { + case <-writer.started: + case <-time.After(2 * time.Second): + require.FailNow(t, "metadata write did not start") + } + node.Name = "pending" + b.updateLocalPeerMetadata(node) + + require.NoError(t, client.Close()) + waitForMetadataSender(t, b) +} diff --git a/services/nvpair-ui-broker/telemetry.go b/services/nvpair-ui-broker/telemetry.go index f2282d1cb..7e96b9007 100644 --- a/services/nvpair-ui-broker/telemetry.go +++ b/services/nvpair-ui-broker/telemetry.go @@ -13,8 +13,6 @@ import ( "nvpair-shared/schedulerwire" ) -const maxTelemetryMilliseconds = int64(1<<63 - 1) - type telemetryObservation struct { value noderec.NodeTelemetry receivedAt time.Time @@ -130,24 +128,7 @@ func (c *telemetryCache) Snapshot(now time.Time) []noderec.NodeTelemetry { } func observedTelemetryAt(observation telemetryObservation, now time.Time) noderec.NodeTelemetry { - value := observation.value - if !value.TelemetryValid { - value.MSSince = 0 - return value - } - if value.MSSince < 0 { - value.MSSince = 0 - } - elapsed := now.Sub(observation.receivedAt).Milliseconds() - if elapsed <= 0 { - return value - } - if elapsed > maxTelemetryMilliseconds-value.MSSince { - value.MSSince = maxTelemetryMilliseconds - } else { - value.MSSince += elapsed - } - return value + return observation.value.Aged(now.Sub(observation.receivedAt)) } func (b *Broker) ingestTelemetry(source nodeSource, value noderec.NodeTelemetry) { diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index 48accee12..0b67c63a0 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -33,6 +33,7 @@ import ( "sort" "strconv" "strings" + "time" ) const ( @@ -326,6 +327,9 @@ const ( // cluster-uuid= TXT key, and a consumer that stops receiving this node's mDNS // record keeps its last observed value indefinitely. MethodSetClusterIdentity = "nodeinfo:set-cluster-identity" + MethodSetPeerMetadata = "nodeinfo:set-peer-metadata" + NotifyInfoUpdated = "nodeinfo:info-updated" + MethodLocalNodeInfo = "discovery:local-node-info" // NotifyObservedAddresses is nvpair-node-info -> broker: the local addresses // peers have actually reached this node on, learned from its own accepted @@ -447,6 +451,61 @@ type MemoryInfo struct { UsedBytes uint64 `json:"used_bytes,omitempty"` } +// NodeInfo is a hardware snapshot. Its identity fields do not establish source +// ownership; the receiving service must establish that through IPC or pinned TLS. +type NodeInfo struct { + GPUs []GPUInfo `json:"GPUs"` + CPU *CPUInfo `json:"cpu,omitempty"` + Memory *MemoryInfo `json:"memory,omitempty"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` + HostUUID string `json:"hostUuid,omitempty"` + ClusterUUID *string `json:"clusterUuid,omitempty"` +} + +// Aged returns a copy whose sample age includes time spent in a queue or cache. +// It preserves validity and clamps age to avoid negative values or overflow. +func (info NodeInfo) Aged(elapsed time.Duration) NodeInfo { + info.MSSince = agedMilliseconds(info.TelemetryValid, info.MSSince, elapsed) + return info +} + +func agedMilliseconds(valid bool, age int64, elapsed time.Duration) int64 { + if !valid { + return 0 + } + if age < 0 { + age = 0 + } + const maxAge = int64(1<<63 - 1) + if ms := elapsed.Milliseconds(); ms > 0 { + if ms > maxAge-age { + return maxAge + } + age += ms + } + return age +} + +// LocalNodeInfoParams is sent only on owned worker IPC, never from a client or +// from a discovered endpoint. The scanner anchors it to its local registry. +type LocalNodeInfoParams struct { + Info NodeInfo `json:"info"` +} + +// PeerMetadata comes from this host's local scanner registry, not mDNS input. +type PeerMetadata struct { + Name string `json:"name"` + Addresses []string `json:"addresses"` + Services map[ServiceKey]int `json:"services"` +} + +type PeerMetadataParams struct { + HostUUID string `json:"hostUuid"` + ClusterUUID string `json:"clusterUuid"` + Peer PeerMetadata `json:"peer"` +} + // ProbeStatus is a service's per-service liveness facet. Liveness is kept // distinct from trust: an untrusted gated service (a 403/TLS failure against a // peer we hold no pin for) is "inaccessible", never "unreachable" — only a @@ -658,3 +717,9 @@ type NodeTelemetry struct { TelemetryValid bool `json:"telemetryValid"` MSSince int64 `json:"msSince"` } + +// Aged advances the sample age without changing its identity or measurements. +func (telemetry NodeTelemetry) Aged(elapsed time.Duration) NodeTelemetry { + telemetry.MSSince = agedMilliseconds(telemetry.TelemetryValid, telemetry.MSSince, elapsed) + return telemetry +} diff --git a/services/shared/noderec/noderec_test.go b/services/shared/noderec/noderec_test.go index 62414cf2f..45b6f0fde 100644 --- a/services/shared/noderec/noderec_test.go +++ b/services/shared/noderec/noderec_test.go @@ -7,11 +7,34 @@ import ( "encoding/json" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestSampleAging_PreservesValidityAndFreshness(t *testing.T) { + // Aging must retain sample validity and add elapsed time without making + // negative ages, backward clock movement, or overflow produce fresh samples. + test := func(name string, info NodeInfo, elapsed time.Duration, wantAge int64) { + t.Run(name, func(t *testing.T) { + got := info.Aged(elapsed) + assert.Equal(t, info.TelemetryValid, got.TelemetryValid) + assert.Equal(t, wantAge, got.MSSince) + + telemetry := NodeTelemetry{HostUUID: "host-a", GPUUtilizationPct: 84, TelemetryValid: info.TelemetryValid, MSSince: info.MSSince} + want := telemetry + want.MSSince = wantAge + assert.Equal(t, want, telemetry.Aged(elapsed), "compact scheduling samples share the hardware age rules") + }) + } + test("queue time", NodeInfo{TelemetryValid: true, MSSince: 137}, 250*time.Millisecond, 387) + test("unavailable sample", NodeInfo{MSSince: 137}, time.Second, 0) + test("negative source age", NodeInfo{TelemetryValid: true, MSSince: -1}, time.Second, 1000) + test("clock moves backwards", NodeInfo{TelemetryValid: true, MSSince: 137}, -time.Second, 137) + test("age saturates", NodeInfo{TelemetryValid: true, MSSince: 1<<63 - 2}, time.Second, 1<<63-1) +} + func TestEngineModels_PrefersPerEngineInventoryAndFallsBackToLegacyModels(t *testing.T) { // Attribution present: each engine gets exactly its own list, never the union. attributed := DirectoryNode{ From 2bc91b12261a6ee449f20f59a54aafe4bf41923a Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Mon, 19 Oct 2026 15:54:12 -0400 Subject: [PATCH 3/8] Serve public and pinned node-info inventory on one port Keep public HTTP hardware displays available while clustered. Peer node inventory over pinned mTLS adds local registry metadata after live membership, exact client pin, and serving identity checks. The broker enables both handlers through the existing cluster directory. Suggested review order: - services/nvpair-node-info/spec.md, services/nvpair-node-scanner/spec.md, then docs/architecture.mdx - transport and inventory contracts. - services/shared/noderec/noderec_test.go, then noderec.go - peer metadata is optional and absent from public hardware responses. - services/nvpair-node-info/main_test.go and cluster_mtls_test.go - public and protected handlers, registry metadata, reused connections, and revocation during collection. Include the typed-response fixture adjustments in cluster_identity_test.go and stats_test.go. - services/nvpair-node-info/main.go - independent public and protected handlers share the registered port, with live pin and metadata identity checks. - services/nvpair-ui-broker/broker.go, nodeinfo.go, and discovery.go, plus deleted nodeinfo_identity_test.go - pass the cluster directory and replace the broker identity push with node-info's live membership reads. - Node-info, scanner, and broker READMEs, including listener modes and CLI help. Validation: affected Go suites, desktop unit and static checks, three cross-process TLS compatibility tests, all 13 Windows binary builds, and SPDX checks passed. Two existing Go tests and two desktop tests remain platform skips. Focused peer node inventory and wire-format tests, CLI help, formatting, and whitespace checks also passed after the review edits. Signed-off-by: Kaylee Lubick --- docs/architecture.mdx | 41 +-- services/nvpair-node-info/README.md | 63 +++-- .../nvpair-node-info/cluster_identity_test.go | 4 +- .../nvpair-node-info/cluster_mtls_test.go | 82 ++++-- services/nvpair-node-info/main.go | 235 +++++++++--------- services/nvpair-node-info/main_test.go | 207 ++++++++++++++- services/nvpair-node-info/spec.md | 8 +- services/nvpair-node-info/stats_test.go | 9 +- services/nvpair-node-scanner/README.md | 2 +- services/nvpair-ui-broker/README.md | 24 +- services/nvpair-ui-broker/broker.go | 87 +------ services/nvpair-ui-broker/discovery.go | 26 -- services/nvpair-ui-broker/nodeinfo.go | 13 +- .../nodeinfo_identity_test.go | 56 ----- services/shared/noderec/noderec.go | 2 + services/shared/noderec/noderec_test.go | 18 ++ 16 files changed, 530 insertions(+), 347 deletions(-) delete mode 100644 services/nvpair-ui-broker/nodeinfo_identity_test.go diff --git a/docs/architecture.mdx b/docs/architecture.mdx index dcb2084f8..0dc74e872 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -171,17 +171,22 @@ Not every surface is gated, and the exceptions are deliberate: | Workload replication, error synchronization | Mutual TLS | | Cluster membership | Mutual TLS, after pairing | | Pairing itself | Plaintext, authenticated by the PIN | -| Host and GPU telemetry | Plaintext HTTP, **not** authenticated | +| Host and GPU hardware displays | Plaintext HTTP, **not** authenticated | +| Peer node inventory and registry metadata | Pinned mutual TLS on the node-info port | Pairing is plaintext because no trust exists yet at that point. The six-digit PIN is what authenticates the exchange that establishes the pinned certificates. -Node telemetry is a considered trade-off, not an oversight. It is treated as the -lowest-sensitivity inter-node surface and is read by consumers that are not -cluster members, so it stays plaintext even on a clustered node. The practical -consequence is that hostname, hardware inventory, and utilization for a node -running PAIR are readable by anything on the same subnet that asks. Do not run -PAIR on a network where that is unacceptable. +Public hardware displays remain available over unauthenticated HTTP, including +while clustered. Any machine that can reach this endpoint can read the node's +hardware inventory and utilization. Do not run PAIR on a network where that is +unacceptable. + +The same node-info port also serves peer node inventory over pinned mutual +TLS. This response includes the serving host's local registry metadata. It requires +current membership, the client's exact approved certificate, and registry +metadata matching the serving identity. Public HTTP continues to return the +hardware display response independently of this peer node inventory handler. ### Two Personalities on One Port @@ -540,7 +545,7 @@ A default installation listens on these ports: | `11434` | Ollama-compatible proxy (Ollama itself moves to `11435`+) | | `1234` | OpenAI-compatible proxy (LM Studio moves to `1235`+) | | `8080` | OpenAI-compatible llama.cpp proxy (managed router on `8081`) | -| `14318` | Node hardware and telemetry | +| `14318` | Public hardware displays and peer node inventory | | `14319` | Service-error synchronization between nodes | | `14320` | Workload propagation between nodes | | `14321` | Pairing and cluster membership | @@ -619,19 +624,27 @@ long-lived receiver. ### Node Enrichment -Announcements establish that a node exists and which ports it serves. PAIR -fetches the interesting facts afterwards, for each node, over HTTP: +Announcements supply discovery candidates and advertised service ports. Scanner +enriches node records with details from two services: + +- GPU, CPU, and memory inventory from node-info, currently over public HTTP for + remote nodes and private worker IPC for the local host +- Its model list from engine-manager, over pinned mutual TLS for remote peers or + loopback HTTP for the local host, as a flat union, a per-engine breakdown, and + the per-engine set of models currently loaded in memory -- GPU, CPU, and memory inventory from its node-info port -- Its model list from its engine-manager port, as a flat union, a per-engine - breakdown, and the per-engine set of models currently loaded in memory +The peer node inventory handler is available, but migrating remote hardware +reads to pinned mutual TLS and promoting verified records remain pending work +in the [scanner specification](../services/nvpair-node-scanner/spec.md). Each enrichment keeps a last-good value, so one failed fetch dims nothing. A node's card does not blank out because a single poll timed out. With a cluster directory configured, a browsed peer whose `cluster-uuid=` matches a pin this node holds is annotated as trusted, which is how the interface can -distinguish a paired node from a stranger it can merely see. +distinguish a paired node from a stranger it can merely see. This annotation +matches an advertised identity to a saved pin; it does not authenticate the +announcing endpoint. ### Node Eviction diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index 2b2484cc1..fb86f2479 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -5,13 +5,13 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-node-info -A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical memory) over a small HTTP API at `/v1/node-info`. It advertises nothing over mDNS itself — its parent (the broker) registers its `ni` port with the `nvpair-node-scanner` discovery daemon, which folds it into this node's single `_nvpair-node` record and fetches `/v1/node-info` to enrich the node for peers. +A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical memory) over a small HTTP API at `/v1/node-info`. It advertises nothing over mDNS itself — its parent (the broker) registers its `ni` port with the `nvpair-node-scanner` discovery daemon, which folds it into this node's single `_nvpair-node` record. ## Communication Two surfaces: -- **HTTP(S)** — serves the node inventory at `/v1/node-info`. Plaintext HTTP on `:14318` by default; optional HTTPS (with optional mTLS) on `:14319` when a cert/key pair is supplied. +- **HTTP(S)** — serves inventory at `/v1/node-info`. With `--cluster-dir`, public HTTP and pinned mTLS share `:14318`. Without it, the default is public HTTP on `:14318`; operator-supplied certificates enable a separate HTTPS listener on `:14319`, with optional CA-based client authentication. - **stdio JSON-RPC 2.0** — newline-delimited, used for local inventory snapshots, registry metadata, lifecycle/control, and shutdown via stdin EOF. The service is normally launched as a subprocess by the broker. It does **not** advertise itself over mDNS. Discovery is centralized in the @@ -34,25 +34,28 @@ scanner registry snapshot over stdin, with `{ok:true}` for an id-bearing request `peer` contains `{name, addresses, services}`; `services` maps service keys to ports. A mismatched host identity or malformed payload returns invalid params (`-32602`). Each accepted snapshot replaces the previous one completely, so a -removed service or address is cleared. An empty `clusterUuid` explicitly clears -membership. The broker refreshes this metadata on local registry changes and -resends it after node-info restarts. Metadata is retained for the protected -inventory handler described in scanner's spec; it is not added to public HTTP. +removed service or address is cleared. An empty `clusterUuid` clears the +membership claim in this snapshot; with `--cluster-dir`, actual membership still +comes from the trust store. The broker refreshes this metadata on local registry +changes and resends it after node-info restarts. Metadata is served only by the +peer node inventory handler; it is not added to public HTTP or local hardware +snapshots. -`nodeinfo:set-cluster-identity {clusterUuid}` continues to update the public -membership field. `log/set-level` is also accepted. +Without `--cluster-dir`, `nodeinfo:set-cluster-identity {clusterUuid}` updates +the public membership field. With a cluster directory, live trust-store state +supplies that field instead. `log/set-level` is also accepted. ## CLI Flags | Flag | Default | Description | |---|---|---| -| `--port ` | `14318` | HTTP port to listen on | +| `--port ` | `14318` | Primary port: public HTTP and, with `--cluster-dir`, pinned mTLS for peer node inventory | | `--tls-port ` | `14319` | HTTPS port (only used when `--cert` and `--key` are set) | | `--cert ` | _(none)_ | TLS server certificate (PEM); enables HTTPS when set together with `--key` | | `--key ` | _(none)_ | TLS server private key (PEM); enables HTTPS when set together with `--cert` | -| `--client-ca ` | _(none)_ | PEM bundle of CAs trusted to sign client certificates; enables mTLS when set | -| `--accept-http` | `false` | Keep the plaintext HTTP listener alive even when TLS is enabled (default: HTTPS-only once `--cert` is set) | -| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`); when set, the primary port carries one cluster-gated listener. While this node is a cluster member, `/v1/node-info` is served **only** over cluster-scoped pin-based mTLS, to pinned cluster peers — or this node itself via self-trust — and a plaintext caller is refused with `403`; while it is not a member the plaintext inventory is served as usual. Membership is re-read per request. Takes precedence over the `--cert` path | +| `--client-ca ` | _(none)_ | PEM bundle of CAs trusted to sign client certificates for operator-configured HTTPS; requires `--cert` and `--key` | +| `--accept-http` | `false` | Keep plaintext HTTP alongside operator-configured HTTPS. With `--cluster-dir`, public HTTP is always available | +| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`). The primary port serves public HTTP hardware displays alongside peer node inventory over pinned mTLS. Public HTTP stays available while clustered. Peer node inventory reads require live membership and matching local registry metadata. Takes precedence over the operator TLS flags | | `--version` | | Print version and exit | `--log-level` is also accepted (registered by `nvpair-shared/applog`). @@ -65,9 +68,17 @@ The cert flags follow a bring-your-own contract, validated at startup: - `--client-ca` requires `--cert` and `--key`. When set, the server requires and verifies a client certificate (`RequireAndVerifyClientCert`); connections without a trusted client cert are dropped at handshake time. - TLS minimum version is pinned to TLS 1.2. -When TLS is enabled, the HTTP listener is dropped unless `--accept-http` is passed. Both listeners share the same handler, but bind separate ports so clients can migrate independently. +With operator-supplied `--cert`/`--key` and no `--cluster-dir`, the HTTP listener +is dropped unless `--accept-http` is passed. Both listeners share the public +inventory handler, but bind separate ports so clients can migrate independently. -`--cluster-dir` replaces both with a single cluster-gated listener on the primary port (see the flag table): one bound port, both personalities, and the choice between them follows live cluster membership rather than being fixed when the listener was bound. +`--cluster-dir` replaces both with one split listener on the primary port. Its +public HTTP handler is independent of membership. Its mTLS handler verifies the +client's exact current pin on every request, including reused connections. A +retained keypair after departure does not keep peer node inventory available. +Operator certificates or CA settings cannot replace those cluster pin checks. +Without a cluster directory, the operator HTTPS listener serves the public +inventory shape and omits peer registry metadata. ## HTTP API @@ -106,14 +117,34 @@ Field notes: - Dynamic fields (`vram_used_bytes`, `utilization_percent`, `cpu.utilization_percent`, `memory.used_bytes`) are populated by the Windows, Linux, and macOS per-tick stats collectors. - `telemetryValid` and `msSince` describe the node-wide GPU utilization snapshot. `telemetryValid` is `true` after the collector has produced a usable GPU sample; `msSince` is that sample's age in milliseconds at response time. A failed collection retains the last usable sample and lets its age increase. Before the first usable sample, and on platforms without dynamic GPU telemetry, the response reports `telemetryValid:false` and `msSince:0`; consumers must ignore the age while validity is false. - `clusterUuid` is the cluster principal this node currently holds. It has three distinct states on the wire: **absent** means unknown, **present and empty** means this node belongs to no cluster, and a value is that principal. A consumer must not read absent as unclustered — that is how a node too old to report the field answers, and also how this node answers before its parent has told it anything, so acting on it would clear a correct annotation elsewhere in the fleet. -- Under the broker, `clusterUuid` is pushed in over stdin (`nodeinfo:set-cluster-identity`) because node-info is spawned with no cluster dir and so cannot read membership itself; the field stays absent until the first push arrives. Standalone with `--cluster-dir`, it reads membership from the trust store per request instead and is therefore always known. The two sources are mutually exclusive by deployment, not a fallback chain. +- With `--cluster-dir`, including under the broker, `clusterUuid` comes from live trust-store membership. Without a cluster directory, a parent may push it over stdin using `nodeinfo:set-cluster-identity`; the field stays absent until that first push. These are separate deployment modes. - `clusterUuid` exists so a peer can learn this node's membership without its mDNS record. Membership otherwise travels only as the `cluster-uuid=` TXT key, which a consumer reads once per record *change*; a consumer that misses that change keeps the previous value indefinitely, and one still holding a departed node's principal will suppress the invite that would bring it back. - All dynamic fields and the `cpu` / `memory` objects use `omitempty`: a value the service couldn't read is dropped from the JSON entirely rather than reported as a misleading literal zero. A genuinely idle CPU renders the same as "unknown" — that ambiguity is intentional and benign. - `vram_bytes` is reported through DXGI on Windows, `nvidia-smi` on Linux, and IORegistry on macOS. On a unified-memory NVIDIA GPU such as DGX Spark, Linux uses total physical system memory for `vram_bytes` and the independently sampled system-memory usage for `vram_used_bytes`. On Apple Silicon, `vram_bytes` is total physical unified memory and `vram_used_bytes` is the GPU driver's mapped allocation (`Alloc system memory`), not whole-system RAM usage or the momentarily active subset. +### Peer node inventory on the same port + +With `--cluster-dir`, pinned peers read the same route over mTLS. The response +adds `peer: {name, addresses, services}` from the owned scanner registry, where +`services` maps service keys to ports. Some of these details are also advertised +over mDNS; mTLS supplies an authenticated registry snapshot. +`hostUuid`, `clusterUuid`, and registry identity must agree with this node's +current certificate identity. The handler returns `503` until matching metadata, +a name, addresses, and the `ni` service registration are available. + +The serving node must still be a member, and the client's presented certificate +must match its exact current pin. Revoked or replaced pins return `403`, including +on reused TLS connections and when trust changes during response collection. +Departure disables peer node inventory even if the keypair remains. +Public HTTP remains available and omits `peer` in every membership state. + ## Discovery -The service no longer advertises itself over mDNS. Its parent (the broker) registers the `ni` port with the `nvpair-node-scanner` discovery daemon, which carries it on this node's single `_nvpair-node` record; a peer's daemon then fetches `/v1/node-info` over plain HTTP to enrich the node. Transport is derived from the shared per-service policy (`nvpair-shared/noderec`): node-info is served plain on the broker path. The standalone BYO-TLS / `--cluster-dir` mTLS serving above is retained for running node-info on its own, but is unused under the broker. +The broker registers the `ni` port with scanner, which carries it on this host's +single `_nvpair-node` record. The broker supplies the cluster directory so this +port supports both public display HTTP and peer node inventory over pinned +mTLS. Local service measurements continue to use the owned worker's private IPC +snapshots. ## Platform Notes diff --git a/services/nvpair-node-info/cluster_identity_test.go b/services/nvpair-node-info/cluster_identity_test.go index 9109c732c..d05302505 100644 --- a/services/nvpair-node-info/cluster_identity_test.go +++ b/services/nvpair-node-info/cluster_identity_test.go @@ -92,8 +92,10 @@ func TestHandleClusterIdentity_IgnoresMalformed(t *testing.T) { func TestBuildResponse_DistinguishesUnknownAndEmptyClusterIdentity(t *testing.T) { raw := func(clusterUUID *string) map[string]any { t.Helper() + body, err := json.Marshal(buildResponse(nil, nil, 0, statsSnapshot{}, "host", clusterUUID)) + require.NoError(t, err) var out map[string]any - require.NoError(t, json.Unmarshal(buildResponse(nil, nil, 0, statsSnapshot{}, "host", clusterUUID), &out), "decode") + require.NoError(t, json.Unmarshal(body, &out), "decode") return out } diff --git a/services/nvpair-node-info/cluster_mtls_test.go b/services/nvpair-node-info/cluster_mtls_test.go index 385da6471..e104f9ee5 100644 --- a/services/nvpair-node-info/cluster_mtls_test.go +++ b/services/nvpair-node-info/cluster_mtls_test.go @@ -20,9 +20,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "nvpair-shared/clustertrust" + "nvpair-shared/noderec" ) // genLeaf mints an Ed25519 self-signed leaf carrying uuid in CN + urn:nvpair:node @@ -71,13 +73,9 @@ func setupNode(t *testing.T, certPEM, keyPEM []byte, pins map[string][]byte) str return dir } -// TestNodeInfoHandler_MTLSGate is the evidence for node-info: the GPU -// inventory is served only to pinned cluster members. A pinned peer is accepted -// (200), a node that completes the handshake but isn't pinned by the server is -// rejected at the gate (403), and — the behavior node-info specifically needs — -// the node can read its OWN inventory over the same mTLS path via self-trust -// (so the local scanner's self-read still works with no plaintext loopback). -func TestNodeInfoHandler_RequiresConfiguredMutualTLS(t *testing.T) { +// Peer node inventory requires a pinned client, including on a new connection. +// Shared mesh self-trust is also accepted; local services use private IPC. +func TestNodeInfoHandler_RequiresPinnedClients(t *testing.T) { aCert, aKey := genLeaf(t, "uuid-a") bCert, bKey := genLeaf(t, "uuid-b") cCert, cKey := genLeaf(t, "uuid-c") @@ -93,15 +91,18 @@ func TestNodeInfoHandler_RequiresConfiguredMutualTLS(t *testing.T) { require.True(t, meshB.Clustered(), "a populated cluster dir must read as clustered") require.True(t, meshC.Clustered(), "a populated cluster dir must read as clustered") - const wantBody = `{"GPUs":[]}` + principal := "uuid-a" + wantInfo := buildResponse(nil, nil, 0, statsSnapshot{}, principal, &principal) + metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318), known: true} mux := http.NewServeMux() - mux.HandleFunc("/v1/node-info", nodeInfoHandler(meshA, func() []byte { return []byte(wantBody) })) + mux.HandleFunc("/v1/node-info", nodeInfoHandler(meshA, metadata, func() NodeInfoResponse { return wantInfo })) srv := httptest.NewUnstartedServer(mux) srv.TLS = meshA.ServerTLSConfig() srv.StartTLS() defer srv.Close() - get := func(m *clustertrust.Mesh, peerUUID string) (int, string) { + get := func(m *clustertrust.Mesh, peerUUID string) (int, noderec.NodeInfo) { + t.Helper() cfg, ok := m.ClientTLSConfig(peerUUID) require.True(t, ok, "no client config for (%v)", peerUUID) client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: cfg}} @@ -110,20 +111,67 @@ func TestNodeInfoHandler_RequiresConfiguredMutualTLS(t *testing.T) { defer func() { require.NoError(t, resp.Body.Close()) }() b, err := io.ReadAll(resp.Body) require.NoError(t, err) - return resp.StatusCode, string(b) + var info noderec.NodeInfo + if resp.StatusCode == http.StatusOK { + require.NoError(t, json.Unmarshal(b, &info)) + } + return resp.StatusCode, info } // Pinned member B -> A: accepted, real body. - code, body := get(meshB, "uuid-a") - require.Equal(t, http.StatusOK, code, "pinned member read") - require.Equal(t, wantBody, body, "pinned member read") + code, info := get(meshB, "uuid-a") + require.Equal(t, http.StatusOK, code) + assert.Equal(t, "uuid-a", info.HostUUID) + assert.NotNil(t, info.Peer) // Self-read A -> A: accepted via self-trust (A isn't in its own trusted/). - code, body = get(meshA, "uuid-a") - require.Equal(t, http.StatusOK, code, "self read") - require.Equal(t, wantBody, body, "self read") + code, info = get(meshA, "uuid-a") + require.Equal(t, http.StatusOK, code) + assert.Equal(t, "uuid-a", info.HostUUID) + assert.NotNil(t, info.Peer) // C completes the handshake (it pins A) but A doesn't pin C -> 403 at the gate. code, _ = get(meshC, "uuid-a") require.Equal(t, http.StatusForbidden, code, "non-member read") } + +func TestNodeInfoHandler_PreservesHardwareForPinnedPeers(t *testing.T) { + // A pinned peer must receive the populated hardware snapshot alongside + // registry metadata, including sample validity and its original age. + serverCert, serverKey := genLeaf(t, "uuid-a") + clientCert, clientKey := genLeaf(t, "uuid-b") + serverDir := setupNode(t, serverCert, serverKey, map[string][]byte{"uuid-b": clientCert}) + clientDir := setupNode(t, clientCert, clientKey, map[string][]byte{"uuid-a": serverCert}) + serverMesh, clientMesh := clustertrust.Open(serverDir), clustertrust.Open(clientDir) + principal := "uuid-a" + snapshot := NodeInfoResponse{ + GPUs: []GPUInfo{{Name: "GPU 0", VramBytes: 48 << 30, VramUsedBytes: 12 << 30, UtilizationPercent: 42}}, + CPU: &CPUInfo{Name: "CPU 0", Cores: 64, UtilizationPercent: 7}, + Memory: &MemoryInfo{TotalBytes: 128 << 30, UsedBytes: 32 << 30}, + TelemetryValid: true, MSSince: 137, + HostUUID: principal, ClusterUUID: &principal, + } + metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318), known: true} + server := httptest.NewUnstartedServer(nodeInfoHandler(serverMesh, metadata, func() NodeInfoResponse { return snapshot })) + server.TLS = serverMesh.ServerTLSConfig() + server.StartTLS() + t.Cleanup(server.Close) + clientTLS, ok := clientMesh.ClientTLSConfig("uuid-a") + require.True(t, ok) + client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: clientTLS}} + t.Cleanup(client.CloseIdleConnections) + + response, err := client.Get(server.URL + "/v1/node-info") + require.NoError(t, err) + defer func() { assert.NoError(t, response.Body.Close()) }() + require.Equal(t, http.StatusOK, response.StatusCode) + var received noderec.NodeInfo + require.NoError(t, json.NewDecoder(response.Body).Decode(&received)) + + assert.Equal(t, []noderec.GPUInfo{{Name: "GPU 0", VramBytes: 48 << 30, VramUsedBytes: 12 << 30, UtilizationPercent: 42}}, received.GPUs) + assert.Equal(t, &noderec.CPUInfo{Name: "CPU 0", Cores: 64, UtilizationPercent: 7}, received.CPU) + assert.Equal(t, &noderec.MemoryInfo{TotalBytes: 128 << 30, UsedBytes: 32 << 30}, received.Memory) + assert.True(t, received.TelemetryValid) + assert.Equal(t, int64(137), received.MSSince) + assert.Equal(t, &metadata.value.Peer, received.Peer) +} diff --git a/services/nvpair-node-info/main.go b/services/nvpair-node-info/main.go index 96f294a48..644954900 100644 --- a/services/nvpair-node-info/main.go +++ b/services/nvpair-node-info/main.go @@ -102,12 +102,15 @@ type NodeInfoResponse struct { ClusterUUID *string `json:"clusterUuid,omitempty"` } -// clusterIdentity is the cluster principal this node reports, kept current by -// the parent broker over stdin (noderec.MethodSetClusterIdentity). node-info -// cannot derive it under the broker: it is spawned without a cluster dir so its -// inventory stays readable by any LAN peer, which also denies it the trust store -// membership is read from. Guarded because the stdin reader and every HTTP -// handler touch it. +type peerNodeInfoResponse struct { + NodeInfoResponse + Peer noderec.PeerMetadata `json:"peer"` +} + +// clusterIdentity holds the principal supplied over private broker IPC for +// launches without a cluster directory. With --cluster-dir, responses use live +// mesh membership instead. Guarded because the stdin reader and HTTP handlers +// can access it concurrently. // // `told` distinguishes "the broker says we hold no principal" from "the broker // has not said anything yet", which are the same empty string but must not be @@ -194,14 +197,14 @@ func handlePeerMetadata(msg applog.StdinMessage, hostUUID string, metadata *peer } type infoUpdateParams struct { - Info json.RawMessage `json:"info"` + Info NodeInfoResponse `json:"info"` } // publishInfoUpdates sends inventory directly to the owning broker at startup // and on each tick until cancellation, using the same builder as public HTTP. -func publishInfoUpdates(ctx context.Context, out *applog.Notifier, body func() []byte, ticks <-chan time.Time) { +func publishInfoUpdates(ctx context.Context, out *applog.Notifier, snapshot func() NodeInfoResponse, ticks <-chan time.Time) { for { - if err := out.Notify(noderec.NotifyInfoUpdated, infoUpdateParams{Info: body()}); err != nil { + if err := out.Notify(noderec.NotifyInfoUpdated, infoUpdateParams{Info: snapshot()}); err != nil { slog.Warn("failed to publish local inventory", "err", err) } select { @@ -220,8 +223,8 @@ func publishInfoUpdates(ctx context.Context, out *applog.Notifier, body func() [ // in stats_windows.go / stats_linux.go / stats_darwin.go / stats_other.go. // buildResponse merges the static identity pulled once at startup with -// the latest dynamic stats snapshot and returns the marshaled -// NodeInfoResponse body. Kept as a free function so unit tests can +// the latest dynamic stats snapshot and returns a typed hardware response. +// HTTP and IPC writers encode it at their boundaries. Unit tests can // exercise the merge without standing up a collector or HTTP server. // // GPUs whose statsKey doesn't appear in snap.GPU (host with no dynamic GPU @@ -234,11 +237,11 @@ func publishInfoUpdates(ctx context.Context, out *applog.Notifier, body func() [ // cpuStatic is nil when static CPU introspection failed; memTotal is zero when // physical-memory introspection failed. Both conditions omit their respective // top-level object from the JSON entirely. -func buildResponse(gpus []GPUInfo, cpuStatic *CPUInfo, memTotal uint64, snap statsSnapshot, hostUUID string, clusterUUID *string) []byte { +func buildResponse(gpus []GPUInfo, cpuStatic *CPUInfo, memTotal uint64, snap statsSnapshot, hostUUID string, clusterUUID *string) NodeInfoResponse { return buildResponseAt(gpus, cpuStatic, memTotal, snap, hostUUID, clusterUUID, time.Now()) } -func buildResponseAt(gpus []GPUInfo, cpuStatic *CPUInfo, memTotal uint64, snap statsSnapshot, hostUUID string, clusterUUID *string, now time.Time) []byte { +func buildResponseAt(gpus []GPUInfo, cpuStatic *CPUInfo, memTotal uint64, snap statsSnapshot, hostUUID string, clusterUUID *string, now time.Time) NodeInfoResponse { outGPUs := mergeGPUInventory(gpus, snap.GPUInventory) for i := range outGPUs { gpu := &outGPUs[i] @@ -272,8 +275,7 @@ func buildResponseAt(gpus []GPUInfo, cpuStatic *CPUInfo, memTotal uint64, snap s UsedBytes: snap.MemUsedBytes, } } - body, _ := json.Marshal(resp) - return body + return resp } func telemetryStatus(sampledAt, now time.Time) (bool, int64) { @@ -315,24 +317,48 @@ func mergeGPUInventory(static, recovered []GPUInfo) []GPUInfo { return merged } -// nodeInfoHandler serves the marshaled node inventory from body(). While this -// node is a cluster member the endpoint is cluster-gated: the authenticated -// client must present a cert pinned for its UUID (or this node's own leaf, for a -// local self-read), or it gets a 403 — so a non-cluster peer can't read this -// host's GPU inventory in the clear, and neither can a plain-HTTP caller on the -// shared port. Refresh picks up a membership change or a peer paired after -// startup. -func nodeInfoHandler(mesh *clustertrust.Mesh, body func() []byte) http.HandlerFunc { +// publicNodeInfoHandler preserves the display response, including while this +// node is clustered. Public readings carry no authoritative peer metadata. +func publicNodeInfoHandler(mesh *clustertrust.Mesh, snapshot func() NodeInfoResponse) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { mesh.Refresh() - if mesh.Clustered() { - if _, ok := mesh.VerifyClientPin(r); !ok { - http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) - return - } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(snapshot()); err != nil { + slog.Warn("failed to serve public node inventory", "err", err) + } + } +} + +// nodeInfoHandler serves peer node inventory only to currently authenticated peers. +// Metadata and hardware must agree with this node's live certificate identity. +func nodeInfoHandler(mesh *clustertrust.Mesh, metadata *peerMetadata, snapshot func() NodeInfoResponse) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mesh.Refresh() + if _, ok := mesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) + return + } + info := snapshot() + // Recheck after building the response: departure or pin replacement must + // also revoke a request that was already in progress on an old connection. + mesh.Refresh() + if _, ok := mesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) + return + } + local, known := metadata.get() + principal := mesh.NodeUUID() + port := local.Peer.Services[noderec.ServiceNodeInfo] + if !known || local.HostUUID != principal || local.ClusterUUID != principal || + info.HostUUID != principal || info.ClusterUUID == nil || *info.ClusterUUID != principal || + local.Peer.Name == "" || len(local.Peer.Addresses) == 0 || port <= 0 || port > 65535 { + http.Error(w, "local peer metadata is not ready", http.StatusServiceUnavailable) + return } w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(body()) + if err := json.NewEncoder(w).Encode(peerNodeInfoResponse{NodeInfoResponse: info, Peer: local.Peer}); err != nil { + slog.Warn("failed to serve peer node inventory", "err", err) + } } } @@ -343,8 +369,7 @@ var Version = "dev" // resolveHostUUID returns the stable per-host UUID node-info reports. An // explicit --node-id (which the broker passes as its own already-resolved // b.nodeID) wins verbatim, so node-info reports exactly the identity the rest of -// the fleet keys on — even when the broker runs on a custom --cluster-dir data -// root while node-info is spawned plaintext with no cluster-dir of its own. +// the fleet keys on, including deployments with a custom cluster directory. // Standalone (no --node-id) it resolves the shared local identity // itself, preferring the --cluster-dir parent when set. func resolveHostUUID(nodeID, clusterDir string) string { @@ -359,13 +384,13 @@ func resolveHostUUID(nodeID, clusterDir string) string { } func main() { - port := flag.Int("port", 14318, "HTTP port to listen on") - tlsPort := flag.Int("tls-port", 14319, "HTTPS port to listen on (only used when --cert and --key are set)") - certPath := flag.String("cert", "", "path to TLS server certificate (PEM); enables HTTPS when set together with --key") - keyPath := flag.String("key", "", "path to TLS server private key (PEM); enables HTTPS when set together with --cert") - clientCAPath := flag.String("client-ca", "", "path to PEM bundle of CAs trusted to sign client certificates; enables mTLS when set") - acceptHTTP := flag.Bool("accept-http", false, "keep the plaintext HTTP listener alive even when TLS is enabled (default: HTTPS-only when --cert is set)") - clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); when set, /v1/node-info is served only over mTLS scoped to pinned cluster peers instead of plaintext") + port := flag.Int("port", 14318, "primary port for public HTTP and, with --cluster-dir, peer node inventory over pinned mTLS") + tlsPort := flag.Int("tls-port", 14319, "operator HTTPS port (requires --cert and --key; ignored with --cluster-dir)") + certPath := flag.String("cert", "", "path to operator TLS server certificate (PEM); requires --key; ignored with --cluster-dir") + keyPath := flag.String("key", "", "path to operator TLS server private key (PEM); requires --cert; ignored with --cluster-dir") + clientCAPath := flag.String("client-ca", "", "PEM CA bundle for operator HTTPS client authentication; requires --cert and --key; ignored with --cluster-dir") + acceptHTTP := flag.Bool("accept-http", false, "keep public HTTP on --port alongside operator HTTPS; --cluster-dir always keeps public HTTP available") + clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); serves public hardware displays over HTTP and peer node inventory over pinned mTLS, both at /v1/node-info on the primary port") nodeID := flag.String("node-id", "", "stable per-host UUID to report on /v1/node-info; when empty it is resolved from the local identity store. The broker passes its own resolved node UUID so both agree even on a custom data root") showVersion := flag.Bool("version", false, "print version and exit") resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo) @@ -384,14 +409,8 @@ func main() { // the address it was added under. hostUUID := resolveHostUUID(*nodeID, *clusterDir) - // With a --cluster-dir, /v1/node-info is served on one port that carries both - // personalities (first-byte split): while this node is a cluster member only - // pin-gated mTLS callers are admitted, closing the plaintext GPU-inventory leak - // to non-cluster peers, and while it is not a member the plaintext inventory is - // served as usual. The membership answer is live, so a node that joins or - // leaves while this process runs changes personality in place. Without a - // --cluster-dir the bring-your-own --cert path and the plaintext default are - // unchanged. + // The cluster listener keeps public HTTP displays available while serving + // peer node inventory to pinned peers over mTLS on the same primary port. mesh := clustertrust.Open(*clusterDir) clusterGated := *clusterDir != "" @@ -404,17 +423,15 @@ func main() { log.Fatalf("invalid TLS configuration: %v", err) } - // httpEnabled: when no cert is configured the only listener the - // service has *is* HTTP, so the flag is meaningless and we don't - // pay attention to it. With TLS on, --accept-http opts into - // keeping the legacy port alive alongside HTTPS for clients that - // haven't migrated. + // httpEnabled controls the separate public HTTP listener used without + // --cluster-dir. It runs by default; operator certificates disable it unless + // --accept-http is set. The cluster directory mode supplies its own public + // HTTP listener through the split listener below. httpEnabled := !tlsOpts.Enabled() || *acceptHTTP if clusterGated { - // The cluster-gated listener below owns *port and serves both - // personalities on it, so there is no separate plaintext listener to - // start. A member admits only pinned peers (the local scanner reads us - // over mTLS via self-trust); a non-member serves the plaintext inventory. + // The split listener below owns *port and serves both public HTTP and + // peer node inventory over pinned mTLS. Skip the separate HTTP listener; + // public HTTP remains available regardless of membership. if tlsOpts.Enabled() { log.Print("--cluster-dir set: ignoring --cert/--key (cluster mTLS takes precedence)") } @@ -448,14 +465,11 @@ func main() { collector := startStatsCollector() defer collector.Stop() - // Where the reported cluster principal comes from is decided by the - // deployment, not by a fallback chain. With --cluster-dir this process holds - // the trust store and reads membership itself, so the answer is always known - // (the handler refreshes the mesh before building the body). Under the broker - // (no cluster dir, so the inventory stays plain for every LAN peer) it cannot - // read membership at all and the broker pushes it instead — until that first - // push the answer is genuinely unknown and the field is omitted. The two - // sources are mutually exclusive by construction. + // With --cluster-dir, clusterUuid comes from the mesh: this node's principal + // while clustered, or an empty string otherwise. HTTP handlers refresh the + // mesh per request, and the watcher below also updates it for TLS handshakes + // and private inventory snapshots. Without a cluster directory, use the + // identity sent over stdin and omit the field until the first update arrives. identity := &clusterIdentity{} clusterPrincipal := func() *string { if !clusterGated { @@ -472,30 +486,25 @@ func main() { return &principal } - responseBody := func() []byte { + snapshot := func() NodeInfoResponse { return buildResponse(gpus, cpu, memTotal, collector.Snapshot(), hostUUID, clusterPrincipal()) } + metadata := &peerMetadata{} mux := http.NewServeMux() - mux.HandleFunc("/v1/node-info", nodeInfoHandler(mesh, responseBody)) - - // Listener layout (set up below depending on flags). Exactly one of the two - // blocks that own *port runs: - // * With --cluster-dir: one cluster-gated listener on *port (default - // 14318) carrying both personalities behind a first-byte split. Which - // callers it admits follows live cluster membership. - // * Otherwise: httpServer on *port when httpEnabled, and, with - // --cert/--key, tlsServer on *tlsPort (default 14319) — a separate port - // so clients can be migrated independently. - // At least one listener is always running (the flag validation above - // guarantees we don't end up with none: --accept-http only matters when TLS - // is on, and the default with no TLS keeps HTTP on). + mux.HandleFunc("/v1/node-info", publicNodeInfoHandler(mesh, snapshot)) + + // With --cluster-dir, one split listener on *port (default 14318) serves + // public HTTP and peer node inventory over pinned mTLS. Otherwise, public + // HTTP uses *port when httpEnabled, and operator HTTPS uses *tlsPort (default + // 14319) when --cert/--key are configured. Operator HTTPS-only mode leaves + // *port unbound. At least one listener is enabled in every valid mode. var httpServer, tlsServer *http.Server - // split is the first-byte dispatcher behind the cluster-gated listener; it is - // closed on shutdown so both sub-listeners unwind. + // split dispatches public HTTP and peer mTLS connections on the primary port. + // Closing it during shutdown releases both sub-listeners. var split *splitlisten.Splitter - // Every listener feeds the same observer: which local address a peer reached - // is evidence regardless of which personality served the request. + // Observe local addresses reached by non-loopback HTTP clients on either + // transport. These reports describe reachability rather than peer identity. observer := newAddressObserver() if httpEnabled { @@ -516,35 +525,42 @@ func main() { }() } - // The cluster-gated listener takes over the *primary* node-info port (*port) - // rather than the separate *tlsPort. That deliberately avoids colliding with - // nvpair-errors, which the broker also spawns on :14319 — node-info's default - // --tls-port. The BYO --cert path keeps its own *tlsPort so a plaintext and a - // TLS listener can coexist there. + // With --cluster-dir, public HTTP and peer node inventory over pinned mTLS + // share the primary node-info port (*port). This avoids colliding with + // nvpair-errors on :14319, node-info's default --tls-port. + // Without --cluster-dir, operator-supplied --cert/--key enable HTTPS on + // *tlsPort; --accept-http keeps public HTTP available on *port. if clusterGated { base, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) if err != nil { log.Fatalf("failed to listen on port %d: %v", *port, err) } split = splitlisten.New(base) - tlsServer = &http.Server{ + peerMux := http.NewServeMux() + peerMux.HandleFunc("/v1/node-info", nodeInfoHandler(mesh, metadata, snapshot)) + httpServer = &http.Server{ Handler: mux, ReadHeaderTimeout: 5 * time.Second, ConnState: observer.connState, IdleTimeout: clustertrust.PeerListenerIdleTimeout, } + tlsServer = &http.Server{ + Handler: peerMux, + ReadHeaderTimeout: 5 * time.Second, + ConnState: observer.connState, + IdleTimeout: clustertrust.PeerListenerIdleTimeout, + } // The cluster leaf is resolved per handshake from the live mesh, so a - // non-member presents none and the handshake is refused; the plain - // sub-listener carries unclustered callers, whom the handler admits only - // while this node is not a member. - serve := func(l net.Listener) { - if err := tlsServer.Serve(l); err != nil && err != http.ErrServerClosed && err != net.ErrClosed { + // non-member presents none and the handshake is refused. The independent + // plain sub-listener continues serving frontend hardware displays. + serve := func(server *http.Server, l net.Listener) { + if err := server.Serve(l); err != nil && err != http.ErrServerClosed && err != net.ErrClosed { log.Fatalf("node-info server error: %v", err) } } - log.Printf("node-info listening on :%d (cluster-gated; clustered=%v)", *port, mesh.Clustered()) - go serve(tls.NewListener(split.TLS(), mesh.ServerTLSConfig())) - go serve(split.Plain()) + log.Printf("node-info listening on :%d (public HTTP and pinned mTLS; clustered=%v)", *port, mesh.Clustered()) + go serve(tlsServer, tls.NewListener(split.TLS(), mesh.ServerTLSConfig())) + go serve(httpServer, split.Plain()) } else if tlsOpts.Enabled() { tlsCfg, err := buildTLSConfig(tlsOpts) if err != nil { @@ -574,22 +590,17 @@ func main() { }() } - // Discovery consolidation: node-info no longer advertises its own - // _nvpair-node-info mDNS record. It's discovered through the node-scanner - // daemon's single _nvpair-node record (the broker registers this node's ni= - // port) and enriched over plain HTTP at that port. node-info is now a - // headless inventory server — the identity/address (uuid=/ip=) it used to - // stamp is carried by the daemon for the whole node. + // The broker registers this service's ni port with scanner, which advertises + // it in this host's single _nvpair-node record. Public HTTP supports hardware + // displays, pinned mTLS serves peer node inventory, and private stdout + // notifications supply local service measurements. ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // Keep membership converging on its own when cluster-gated. Every other gate - // here is refreshed by the request handler, but the TLS personality is chosen - // during the handshake — which happens BEFORE any handler runs — so without an - // independent watch a node that joined while this process ran could not present - // a leaf until some plaintext caller happened to arrive and refresh membership - // as a side effect. That would make the cluster surface unreachable exactly - // when it is supposed to take over. + // Refresh membership and certificate identity even without HTTP requests. + // TLS chooses the server certificate during the handshake, before a handler + // can refresh the mesh. The watcher makes membership changes visible to new + // handshakes and private snapshots without requiring a public HTTP read. if clusterGated { go mesh.Watch(ctx, nil) } @@ -607,15 +618,13 @@ func main() { } }() - // One writer for both directions of stdout traffic: request responses and the - // observed-address reports below share its lock so their frames never - // interleave. + // Serialize RPC replies, inventory snapshots, and observed-address reports + // through one stdout writer so JSON-RPC frames cannot interleave. notifier := applog.NewNotifier(os.Stdout) go observer.reportLoop(ctx.Done(), notifier) infoTicks := time.NewTicker(2 * time.Second) defer infoTicks.Stop() - go publishInfoUpdates(ctx, notifier, responseBody, infoTicks.C) - metadata := &peerMetadata{} + go publishInfoUpdates(ctx, notifier, snapshot, infoTicks.C) go applog.StdinRPC(notifier, func(msg applog.StdinMessage) { handleClusterIdentity(msg, identity) diff --git a/services/nvpair-node-info/main_test.go b/services/nvpair-node-info/main_test.go index fc3ea456c..47737af1e 100644 --- a/services/nvpair-node-info/main_test.go +++ b/services/nvpair-node-info/main_test.go @@ -6,14 +6,20 @@ package main import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/json" "io" "net" "net/http" + "net/http/httptest" + "net/http/httptrace" + "os" "os/exec" "path/filepath" "runtime" "strconv" + "strings" "testing" "time" @@ -21,6 +27,7 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/applog" + "nvpair-shared/clustertrust" "nvpair-shared/jsonrpc" "nvpair-shared/noderec" ) @@ -89,7 +96,7 @@ type nodeInfoTestProcess struct { // startNodeInfoForTest builds and starts node-info on an available port. // It registers cleanup and bounds the child's lifetime so stalled reads end. -func startNodeInfoForTest(t *testing.T, hostUUID string) *nodeInfoTestProcess { +func startNodeInfoForTest(t *testing.T, hostUUID string, extraArgs ...string) *nodeInfoTestProcess { t.Helper() path := filepath.Join(t.TempDir(), "nvpair-node-info") if runtime.GOOS == "windows" { @@ -107,7 +114,8 @@ func startNodeInfoForTest(t *testing.T, hostUUID string) *nodeInfoTestProcess { require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) t.Cleanup(cancel) - cmd := exec.CommandContext(ctx, path, "--node-id", hostUUID, "--port", portString) + args := append([]string{"--node-id", hostUUID, "--port", portString}, extraArgs...) + cmd := exec.CommandContext(ctx, path, args...) stdin, err := cmd.StdinPipe() require.NoError(t, err) stdout, err := cmd.StdoutPipe() @@ -183,14 +191,14 @@ func TestInfoUpdates_PublishesStartupAndPeriodicSnapshots(t *testing.T) { ticks <- time.Now() var output bytes.Buffer var calls int - body := func() []byte { + snapshot := func() NodeInfoResponse { calls++ if calls == 2 { cancel() } return buildResponse(nil, nil, 0, statsSnapshot{}, "owned-host", nil) } - publishInfoUpdates(ctx, applog.NewNotifier(&output), body, ticks) + publishInfoUpdates(ctx, applog.NewNotifier(&output), snapshot, ticks) decoder := json.NewDecoder(&output) for i := 0; i < 2; i++ { var frame jsonrpc.Message @@ -255,3 +263,194 @@ func TestPeerMetadata_RejectsMalformedOrForeignIdentity(t *testing.T) { test("malformed object", `"invalid"`) test("foreign host", `{"hostUuid":"someone-else","peer":{"name":"copied-name"}}`) } + +type pairedNodeInfoProcess struct { + process *nodeInfoTestProcess + clusterDir string + client *http.Client + peerURL string +} + +func startPairedNodeInfoForTest(t *testing.T) pairedNodeInfoProcess { + t.Helper() + serverCert, serverKey := genLeaf(t, "uuid-a") + clientCert, clientKey := genLeaf(t, "uuid-b") + serverDir := setupNode(t, serverCert, serverKey, map[string][]byte{"uuid-b": clientCert}) + // Keep membership active when the client's pin alone is removed or replaced. + require.NoError(t, os.WriteFile(filepath.Join(serverDir, "admission.json"), []byte(`{"clusterId":"test-cluster","epoch":1}`), 0o600)) + clientDir := setupNode(t, clientCert, clientKey, map[string][]byte{"uuid-a": serverCert}) + cfg, ok := clustertrust.Open(clientDir).ClientTLSConfig("uuid-a") + require.True(t, ok, "pinned client configuration must be available") + client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{TLSClientConfig: cfg}} + t.Cleanup(client.CloseIdleConnections) + process := startNodeInfoForTest(t, "uuid-a", "--cluster-dir", serverDir) + process.nextSnapshot(t) // startup has bound both listener personalities + return pairedNodeInfoProcess{process: process, clusterDir: serverDir, client: client, peerURL: strings.Replace(process.inventoryURL, "http://", "https://", 1)} +} + +func nodeInfoMetadataForTest(port int) noderec.PeerMetadataParams { + return noderec.PeerMetadataParams{ + HostUUID: "uuid-a", ClusterUUID: "uuid-a", + Peer: noderec.PeerMetadata{ + Name: "workstation", Addresses: []string{"192.0.2.20"}, + Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port, noderec.ServiceOllama: 11434}, + }, + } +} + +type nodeInfoTestInventory struct { + NodeInfoResponse + Peer *noderec.PeerMetadata `json:"peer,omitempty"` +} + +type nodeInfoHTTPResult struct { + status int + inventory nodeInfoTestInventory + reused bool +} + +func getNodeInfoForTest(t *testing.T, client *http.Client, endpoint string) nodeInfoHTTPResult { + t.Helper() + request, err := http.NewRequest(http.MethodGet, endpoint, nil) + require.NoError(t, err) + var reused bool + trace := &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused = info.Reused }} + response, err := client.Do(request.WithContext(httptrace.WithClientTrace(request.Context(), trace))) + require.NoError(t, err) + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + result := nodeInfoHTTPResult{status: response.StatusCode, reused: reused} + if response.StatusCode == http.StatusOK { + require.NoError(t, json.Unmarshal(body, &result.inventory)) + } + return result +} + +func TestNodeInfo_PublicInventoryRemainsAvailableWhileClustered(t *testing.T) { + // Verify that a clustered node still serves public hardware over HTTP while + // omitting the peer registry metadata served by the mTLS handler. + paired := startPairedNodeInfoForTest(t) + public := &http.Client{Timeout: 3 * time.Second} + t.Cleanup(public.CloseIdleConnections) + result := getNodeInfoForTest(t, public, paired.process.inventoryURL) + assert.Equal(t, http.StatusOK, result.status) + assert.Equal(t, "uuid-a", result.inventory.HostUUID) + assert.Nil(t, result.inventory.Peer) +} + +func TestNodeInfo_PeerNodeInventoryServesCompleteMatchingNodeDetails(t *testing.T) { + // Verify that a pinned client receives inventory with complete registry + // details matching the serving node's certificate identity. + paired := startPairedNodeInfoForTest(t) + metadata := nodeInfoMetadataForTest(paired.process.port) + paired.process.requestPeerMetadata(t, metadata) + result := getNodeInfoForTest(t, paired.client, paired.peerURL) + require.Equal(t, http.StatusOK, result.status) + assert.Equal(t, "uuid-a", result.inventory.HostUUID) + require.NotNil(t, result.inventory.ClusterUUID) + assert.Equal(t, "uuid-a", *result.inventory.ClusterUUID) + assert.Equal(t, &metadata.Peer, result.inventory.Peer) +} + +func TestNodeInfo_PeerNodeInventoryRejectsIncompleteOrMismatchedNodeDetails(t *testing.T) { + // Verify 503 before any node details arrive and when replacement details + // are incomplete or identify a different node. + paired := startPairedNodeInfoForTest(t) + assert.Equal(t, http.StatusServiceUnavailable, getNodeInfoForTest(t, paired.client, paired.peerURL).status) + reject := func(name string, incomplete noderec.PeerMetadataParams) { + t.Helper() + t.Run(name, func(t *testing.T) { + // Replace a complete snapshot so each case also rejects stale details + // left over from a previously valid state. + paired.process.requestPeerMetadata(t, nodeInfoMetadataForTest(paired.process.port)) + paired.process.requestPeerMetadata(t, incomplete) + assert.Equal(t, http.StatusServiceUnavailable, getNodeInfoForTest(t, paired.client, paired.peerURL).status) + }) + } + principal := nodeInfoMetadataForTest(paired.process.port) + principal.ClusterUUID = "different-principal" + reject("mismatched node identity", principal) + + name := nodeInfoMetadataForTest(paired.process.port) + name.Peer.Name = "" + reject("missing name", name) + + addresses := nodeInfoMetadataForTest(paired.process.port) + addresses.Peer.Addresses = nil + reject("missing addresses", addresses) + + service := nodeInfoMetadataForTest(paired.process.port) + delete(service.Peer.Services, noderec.ServiceNodeInfo) + reject("missing node-info registration", service) +} + +func TestNodeInfo_RechecksTrustOnReusedConnections(t *testing.T) { + // Reuse a TLS connection after pin removal, replacement, or local departure. + // The next inventory request must be forbidden while public HTTP still + // serves hardware without a peer object. + test := func(name string, changeTrust func(*testing.T, string)) { + t.Run(name, func(t *testing.T) { + paired := startPairedNodeInfoForTest(t) + paired.process.requestPeerMetadata(t, nodeInfoMetadataForTest(paired.process.port)) + require.Equal(t, http.StatusOK, getNodeInfoForTest(t, paired.client, paired.peerURL).status) + changeTrust(t, paired.clusterDir) + result := getNodeInfoForTest(t, paired.client, paired.peerURL) + assert.True(t, result.reused) + assert.Equal(t, http.StatusForbidden, result.status) + public := &http.Client{Timeout: 3 * time.Second} + t.Cleanup(public.CloseIdleConnections) + result = getNodeInfoForTest(t, public, paired.process.inventoryURL) + assert.Equal(t, http.StatusOK, result.status) + assert.Nil(t, result.inventory.Peer) + }) + } + removePin := func(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.Remove(filepath.Join(dir, "trusted", "uuid-b.json"))) + } + test("pin removal", removePin) + test("pin replacement with the same UUID", func(t *testing.T, dir string) { + t.Helper() + cert, _ := genLeaf(t, "uuid-b") + type trustedPeer struct { + NodeUUID string `json:"nodeUuid"` + CertPEM string `json:"certPem"` + } + data, err := json.Marshal(trustedPeer{NodeUUID: "uuid-b", CertPEM: string(cert)}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "trusted", "uuid-b.json"), data, 0o600)) + }) + test("local departure retaining the keypair", func(t *testing.T, dir string) { + t.Helper() + removePin(t, dir) + require.NoError(t, os.Remove(filepath.Join(dir, "admission.json"))) + }) +} + +func TestNodeInfoHandler_RejectsInventoryIfTrustChangesDuringCollection(t *testing.T) { + // Revoke the client's pin during response collection and verify that the + // in-flight request is rejected instead of serving the completed inventory. + serverCert, serverKey := genLeaf(t, "uuid-a") + clientCert, clientKey := genLeaf(t, "uuid-b") + serverDir := setupNode(t, serverCert, serverKey, map[string][]byte{"uuid-b": clientCert}) + clientDir := setupNode(t, clientCert, clientKey, map[string][]byte{"uuid-a": serverCert}) + mesh := clustertrust.Open(serverDir) + clientConfig, ok := clustertrust.Open(clientDir).ClientTLSConfig("uuid-a") + require.True(t, ok, "client certificate fixture must be available") + require.NotEmpty(t, clientConfig.Certificates) + require.NotNil(t, clientConfig.Certificates[0].Leaf) + metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318), known: true} + // Call the handler synchronously with the authenticated client's TLS state + // so trust can be changed deterministically while it builds the response. + request := httptest.NewRequest(http.MethodGet, "https://localhost/v1/node-info", nil) + request.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{clientConfig.Certificates[0].Leaf}} + principal := "uuid-a" + snapshot := func() NodeInfoResponse { + require.NoError(t, os.Remove(filepath.Join(serverDir, "trusted", "uuid-b.json"))) + return buildResponse(nil, nil, 0, statsSnapshot{}, principal, &principal) + } + response := httptest.NewRecorder() + nodeInfoHandler(mesh, metadata, snapshot)(response, request) + assert.Equal(t, http.StatusForbidden, response.Code) +} diff --git a/services/nvpair-node-info/spec.md b/services/nvpair-node-info/spec.md index cb0599191..c3c51e3bc 100644 --- a/services/nvpair-node-info/spec.md +++ b/services/nvpair-node-info/spec.md @@ -34,7 +34,7 @@ It exists as a separate binary because the readout is inherently platform-specif - **Local enrichment**: startup and two-second `nodeinfo:info-updated` snapshots travel through the broker as `discovery:local-node-info`. Scanner attaches them to its owned registry record, without a loopback HTTP service read. - **Peer enrichment**: a remote scanner authenticates the node-info endpoint over exact-pin mTLS before promoting an announcement. Protected metadata supplies the peer's name, addresses, service ports, and hardware. Failed authentication never triggers public HTTP fallback. - **Desktop telemetry poll**: Electron polls public HTTP every 2000 ms with a 1500 ms per-attempt timeout, existing address failover, and failure backoff capped at 30 s. It updates hardware display fields only, independently of verified broker snapshots. It reads this host over loopback. Public responses cannot change identity, trust, addresses, service ports, or models. -- **Terminal telemetry poll**: TUI polls the open node detail screen over public HTTP at the same cadence and timeout. It retains its response-size limit and refuses redirects. Verified rows require a matching response `hostUuid`; an unverified row's display preview does not turn the response's UUID into verified identity. +- **Terminal telemetry poll**: TUI polls the open node detail screen over public HTTP at the same cadence and timeout. It retains its response-size limit and refuses redirects. Both clients require a matching response `hostUuid` for verified rows, including the local row; a missing or different UUID is rejected. An unverified row's display preview may omit `hostUuid` and never turns the response's UUID into verified identity. - **Scheduler GPU pressure**: scanner uses owned IPC locally and pinned mTLS remotely; manual-node hardware also requires pinned mTLS. They supply maximum GPU utilization with validity and elapsed age through the broker. Unknown or stale data follows the scheduler's existing freshness and pressure rules, and never becomes freshly idle through a failed read or replay. - **Observed local addresses**: a non-loopback request records the real local address it reached. Every 30 s the service sends the current set as `nodeinfo:observed-addresses`; broker relays it to scanner. This is reachability evidence, including public requests, and does not authenticate the requester. - **Manual node identification**: the manual worker probes `14318`, or the configured node-info port, over pinned mTLS every ten seconds. Matching response host/principal and certificate identities allow the broker to join that source to the same verified host. Explicit engine routes remain independent of hardware authentication. @@ -175,7 +175,7 @@ Outbound, the service sends these notifications to its owning parent: Local snapshots have the public hardware shape and omit protected `peer` metadata. Broker forwards them privately to scanner; they are not frontend HTTP readings. ### 7.1 Response schema -`buildResponse` copies the static `[]GPUInfo` detected at startup, folds in any adapters the snapshot's own `GPUInventory` carries (`mergeGPUInventory` enriches a matching `statsKey` in place and appends an unseen one), overlays the latest snapshot by `statsKey`, and marshals. Field semantics: +`buildResponse` copies the static `[]GPUInfo` detected at startup, folds in any adapters the snapshot's own `GPUInventory` carries (`mergeGPUInventory` enriches a matching `statsKey` in place and appends an unseen one), and overlays the latest snapshot by `statsKey`. The resulting typed snapshot is encoded at the HTTP and IPC boundaries. Field semantics: | JSON tag | Go type | `omitempty` | Source | |----------|---------|-------------|--------| @@ -271,7 +271,7 @@ A multi-homed host cannot tell from its own side which of its addresses a given - `nvidia-smi`, when present, is on `PATH` and returns within 3 s; a wedged driver is bounded by the timeout, not waited out. - `/usr/sbin/ioreg` exists on every supported macOS release, runs unprivileged, and returns within 3 s; its `IOAccelerator` `PerformanceStatistics` keys are undocumented and may be renamed by a macOS release. - With a cluster directory, live membership supplies the cluster principal. Standalone without that directory, the parent's identity push is unknown until first supplied. Public claims never override established identity or trust. -- Consumers treat an absent field as unknown and never as zero, ignore `msSince` while `telemetryValid` is `false`, and tolerate a response that carries only `{"GPUs":[],"telemetryValid":false,"msSince":0}`. +- Consumers treat an absent measurement field as unknown and never as zero, and ignore `msSince` while `telemetryValid` is `false`. An unverified frontend preview may accept a response containing only `{"GPUs":[],"telemetryValid":false,"msSince":0}`. A verified frontend row additionally requires its matching `hostUuid`; protected service readers also require the authenticated identity and peer metadata described above. - Discovery is somebody else's job: the record advertising this port is published by the node-scanner daemon, and this process never touches mDNS. ## 12. Failure Modes and Mitigations @@ -285,7 +285,7 @@ A multi-homed host cannot tell from its own side which of its addresses a given - **Unpinned peer reads protected inventory**: the TLS handler returns `403` unless the client has a current matching pin. Public HTTP hardware remains available and omits registry metadata. - **Owned metadata is absent or inconsistent**: protected inventory returns `503` until broker metadata agrees with the current host and cluster principal. Public display and private local snapshots remain available. - **Malformed cluster-identity push in standalone mode**: the payload is dropped with a warning. An absent field stays unknown; public responses do not authorize service membership changes. -- **Address reuse or wrong host answers a poll**: remote service readers reject certificate or response-identity mismatches and never retry public HTTP. Frontend public reads remain hardware display data, and a verified row requires matching response identity. An unverified preview cannot promote a claim. +- **Address reuse or wrong host answers a poll**: remote service readers reject certificate or response-identity mismatches and never retry public HTTP. Frontend public reads remain hardware display data, and a verified row rejects a missing or mismatched response `hostUuid` while trying its remaining addresses. An unverified preview cannot promote a claim. - **Port already bound**: the process cannot serve. → `log.Fatalf` with the port in the message; the broker's supervisor restarts it and the failure is visible in the shared stderr stream. ## 13. Observability diff --git a/services/nvpair-node-info/stats_test.go b/services/nvpair-node-info/stats_test.go index 83f35333b..29ec4dbfb 100644 --- a/services/nvpair-node-info/stats_test.go +++ b/services/nvpair-node-info/stats_test.go @@ -162,7 +162,7 @@ func TestBuildResponse_ReportsTelemetryAgeAndFreshness(t *testing.T) { now := time.Unix(1_700_000_000, 500_000_000) test := func(name string, sampledAt time.Time, wantValid bool, wantAge int64) { t.Run(name, func(t *testing.T) { - body := buildResponseAt( + typed := buildResponseAt( []GPUInfo{{Name: "GPU 0", statsKey: "gpu-a"}}, nil, 0, @@ -174,11 +174,11 @@ func TestBuildResponse_ReportsTelemetryAgeAndFreshness(t *testing.T) { nil, now, ) - var typed NodeInfoResponse - require.NoError(t, json.Unmarshal(body, &typed), "decode response") require.Equal(t, wantValid, typed.TelemetryValid) require.Equal(t, wantAge, typed.MSSince) + body, err := json.Marshal(typed) + require.NoError(t, err) var raw map[string]any require.NoError(t, json.Unmarshal(body, &raw), "decode raw response") require.Contains(t, raw, "telemetryValid", "telemetryValid missing from response") @@ -197,7 +197,8 @@ func TestBuildResponse_ReportsTelemetryAgeAndFreshness(t *testing.T) { // with no "cpu" key at all, not `"cpu":null`. func buildResponseDecode(t *testing.T, static []GPUInfo, cpu *CPUInfo, memTotal uint64, snap statsSnapshot) (NodeInfoResponse, map[string]any) { t.Helper() - body := buildResponse(static, cpu, memTotal, snap, "", nil) + body, err := json.Marshal(buildResponse(static, cpu, memTotal, snap, "", nil)) + require.NoError(t, err) var typed NodeInfoResponse require.NoError(t, json.Unmarshal(body, &typed), "typed decode") var raw map[string]any diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index 17cd8f0d4..fb473ad1f 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -23,7 +23,7 @@ Concretely, the daemon: - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. - **Anti-flap:** a node has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. - **A scan that loses everything is treated as our own fault:** when more than one node is known and a scan returns *none* of them, no node is penalized for it — for up to six consecutive scans (~30s), after which they count normally. Independent machines do not leave the network in the same five-second window, but a process too starved to drain its multicast socket within `scanTimeout` hears silence from all of them at once; without this, one saturated machine dropped its entire cluster, idle peers included, in a single reconcile. The suppression needs two or more known nodes to engage: losing a lone peer looks the same whether it left or we stopped listening, so that case falls through to the ordinary threshold. -- **Converges without mDNS:** a 15s sweep re-reads each peer over HTTP and folds any change into the directory, independent of browse events. It covers facts that change without moving the mDNS record (a model pull/delete, a late engine start) and facts that did move it but whose announcement was never received. The second case is why a peer's cluster membership is re-read from its `ni` port (`clusterUuid` on `/v1/node-info`, which under the broker is served plain — so it answers whether or not the two nodes share a cluster; a node-info run standalone with `--cluster-dir` gates the endpoint while clustered and will not answer a non-pinned peer, which the sweep treats as no answer): membership otherwise arrives solely as the `cluster-uuid=` TXT key, read once per record change, and the anti-flap probe above deliberately keeps a still-reachable peer's entry alive — so a missed re-advertisement would leave a departed peer's principal in place for the life of the process, suppressing the invite that would bring it back. Membership is refreshed before models on each tick, since the model fetch is pinned mTLS keyed on that principal. Only a peer's explicit report is acted on: a failed fetch, or a peer too old to report the field, leaves the annotation untouched rather than inferring "unclustered" from silence. +- **Converges without mDNS:** a 15s sweep currently re-reads each peer over public HTTP and folds changes into the directory, independent of browse events. This covers missed announcements and facts that change without moving the mDNS record. `clusterUuid` remains available on public node-info, including when its separate peer node inventory handler is enabled. Membership is refreshed before models, because model reads use pinned mTLS keyed on that principal. Failed or absent membership reports leave the annotation unchanged. Migrating these remote inventory reads to authenticated promotion is the next implementation step in [the spec](spec.md). ## Communication diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index b8f7f3fcd..5da30cbe0 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: Apache-2.0 The [engine settings protocol](ENGINE_SETTINGS.md) documents local/paired launch and port editing, revision checks, serialized application and recovery. -A Go service that exposes the NVIDIA Personal AI Router API to **UI processes** +A Go service that exposes the NVIDIA PAIR API to **UI processes** and other clients (CLIs, dashboards, mobile companions, scripts, tests, etc.). The shipped graphical UI is bundled alongside the backend services and launches the broker from the same installation directory. The broker is the parent @@ -22,7 +22,7 @@ namespace: | Worker | Role | Client namespace | | --- | --- | --- | | `nvpair-node-scanner` | Discovery daemon: advertises this host's one `_nvpair-node._tcp` record and browses the LAN | `discovery:*` | -| `nvpair-node-info` | Local GPU / CPU / memory inventory over HTTP at `/v1/node-info` | — (HTTP only) | +| `nvpair-node-info` | Owned inventory IPC, public display HTTP, and peer node inventory over pinned mTLS at `/v1/node-info` | _internal_ | | `nvpair-proxy` | One process hosting an inference proxy and router facade per enabled engine | `ollama-proxy:*`, `lmstudio-proxy:*`, `llamacpp-proxy:*` | | `nvpair-engine-manager` | Local engine and model control plane; also serves `GET /v1/models` to peers | `engine:*` | | `nvpair-cluster-manager` | Node identity, trusted-node store, PIN pairing | `cluster:*`, `nodes:*` | @@ -96,7 +96,7 @@ worker IPC; desktop and TUI continue using public HTTP for hardware displays. | `--settings-path ` | `./nvpair-node-settings[.exe]` in the CWD | Explicit path to the `nvpair-node-settings` binary the broker spawns for the typed settings store. Same optional semantics as `--node-info-path` | | `--cluster-manager-path ` | `./nvpair-cluster-manager[.exe]` in the CWD | Explicit path to the `nvpair-cluster-manager` binary the broker spawns for cluster pairing / membership. Same optional semantics as `--node-info-path` | | `--scheduler-path ` | `./nvpair-job-scheduler[.exe]` in the CWD | Explicit path to the `nvpair-job-scheduler` binary the broker spawns for responsive, node-wide workload and GPU-pressure ranking. Same optional semantics as `--node-info-path` | -| `--cluster-dir ` | `cluster/` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) | Cluster config dir (`node.crt`/`node.key` + `trusted/`, minted by `nvpair-cluster-manager`). Threaded to the cluster-scoped workers (`nvpair-errors`, `nvpair-workload-manager`, `nvpair-node-scanner`, `nvpair-manual-nodes`, and `nvpair-engine-manager`), each of which derives its membership from it continuously — so a create, join, or leave takes effect in place and the broker does **not** restart them. The broker also passes the parent of this path to `nvpair-cluster-manager` as `--config-dir`, so the only writer of the cluster dir and the workers reading it cannot resolve different directories. `nvpair-node-info` is excluded — it stays plain HTTP even when clustered (see the repository-root `SECURITY.md`). Defaults so cluster mTLS auto-activates with nothing to pass; with an empty or cert-less dir this node is not a member, so it serves and dials no inter-node cluster traffic at all | +| `--cluster-dir ` | `cluster/` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) | Cluster identity and pins, threaded to the cluster-scoped workers including node-info. Membership changes take effect without worker restarts. The cluster manager receives its parent as `--config-dir` so readers and the sole writer agree on one directory. Node-info keeps public display HTTP available independently of its peer node inventory handler | | `--log-level ` | _(env `NVPAIR_LOG_LEVEL` or `info`)_ | `debug` \| `info` \| `warn` \| `error` | | `--version` | | Print version and exit | @@ -108,7 +108,11 @@ On startup — **before** emitting `app:ready` — the broker spawns the scanner **`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by `id`. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm`/`lc` from the engine poller) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. -**`nvpair-node-info`** is spawned next. It's a server, not an event source: it stands up the local `/v1/node-info` HTTP endpoint (GPU/CPU/memory inventory). It does not advertise itself — the broker registers its `ni` port with the scanner daemon, which carries it in the node record, and a peer's daemon fetches `/v1/node-info` over plain HTTP to enrich the node. The broker doesn't read anything back from node-info's stdout (drained and discarded). Spawning it is **optional**: if the binary can't be resolved (and no `--node-info-path` override was given) the broker logs a warning and continues serving discovery without it. +**`nvpair-node-info`** serves public display HTTP and peer node inventory over +pinned mTLS on its registered `ni` port. The broker passes its cluster directory and host UUID, +relays owned hardware snapshots from stdout to scanner, and sends local registry +metadata back for peer node inventory responses. Spawning it is optional: if no binary +is resolved and no explicit path was given, the broker continues without it. **Engine advertising.** The broker runs an internal 5 s poll loop against local Ollama (`GET /`), LM Studio (`GET /v1/models`), and llama.cpp (`GET /health`) @@ -641,10 +645,18 @@ Attach to a pre-existing endpoint: `lm`, or `lc` with the daemon while each is up. There's no manual-advertise RPC (custom service, port, name, or TXT), and no way to advertise anything other than the detected engines. -- **node-info control surface.** node-info is spawned and torn down with the broker, and the broker pushes it only two things over stdin: the log level, and this node's cluster principal (`nodeinfo:set-cluster-identity`, sent on spawn and on every membership or pin-set change, because node-info holds no cluster dir and so cannot read membership itself). Otherwise it's hands-off: the broker registers its port with the daemon (which enriches over plain HTTP) but doesn't pass through TLS material (`--cert` / `--key` / `--client-ca`) or a custom `--port`, and exposes no RPC to query or reconfigure it. It runs with its own defaults plus those two pushes. +- **node-info control surface.** The broker passes node-info its cluster directory + and host UUID, relays hardware snapshots from stdout, and sends registry + metadata and log-level updates over stdin. Membership comes from the live trust + store. The broker registers the fixed `ni` port and + exposes no client RPC to query or reconfigure node-info, or to supply operator + TLS flags or a custom port. - **Manual-node persistence across restarts.** `nvpair-manual-nodes` remains an in-memory worker; the broker owns `configs/manual-nodes.json`, clears stale live projections when the worker exits, and replays the durable entries into its replacement. - **Per-event push semantics.** `discovery:nodes-changed` always carries the full current snapshot, not a delta. For small N this is fine and lets the client treat the payload as authoritative without state reconciliation. `errors:update` is likewise a full snapshot. -- **BYO-TLS node-info discovery.** The daemon enriches peers over plain HTTP per the consolidated transport policy (node-info is plain on the broker path). Reading a node-info served over operator-configured BYO-TLS / mTLS is not supported through the broker; discovery works for the plain-HTTP case (the common one). +- **BYO-TLS node-info discovery.** The broker does not expose scanner's operator + TLS flags. Scanner's remote hardware reads currently use public HTTP, even + though node-info also offers peer node inventory over pinned mTLS. + Migrating those reads remains pending work in the [scanner spec](../nvpair-node-scanner/spec.md). - **Per-worker log-level granularity.** `log/set-level` fans the broker's single level out to every running child, but there's no way to set a different level per worker — it's one level for the whole process tree. - **AuthN / AuthZ.** Transport security relies on the parent's pipe / socket ACL. There is no per-message token. - **HTTP / WebSocket transport.** JSON-RPC over stdio or pipe only — no web bridge yet. Likely lives in a separate component if/when added. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 5c0579864..9e44ec13a 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -20,7 +20,6 @@ import ( "nvpair-shared/appdir" "nvpair-shared/applog" - "nvpair-shared/clustertrust" "nvpair-shared/engines" "nvpair-shared/errors" "nvpair-shared/nodeid" @@ -646,27 +645,17 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { return sp, nil } -// spawnNodeInfo starts the owned inventory worker, restores identity and peer -// metadata, and registers its service port with scanner. +// spawnNodeInfo starts the owned inventory worker, restores peer metadata, +// and registers its service port with scanner. func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { - // Keep public HTTP available for desktop and TUI hardware displays. The - // current --cluster-dir mode gates that endpoint while clustered, so do not - // enable it here. Local service measurements use the owned worker's stdio, - // independent of whether this host belongs to a cluster. - // - // Do pass --node-id: node-info reports the broker's already-resolved UUID so - // /v1/node-info agrees with the identity the fleet keys on, even on a custom - // --cluster-dir data root (node-info gets no cluster-dir, so it would - // otherwise resolve the default root and could report a different UUID). - np, err := startNodeInfo(b.nodeInfoPath, applog.LevelString(), b.forwardNodeInfoNotification, "--node-id", b.nodeID) + // One port serves public display HTTP and pinned peer mTLS. Local service + // measurements still use the owned worker's stdio, independent of membership. + extra := append(b.clusterDirArgs(), "--node-id", b.nodeID) + np, err := startNodeInfo(b.nodeInfoPath, applog.LevelString(), b.forwardNodeInfoNotification, extra...) if err != nil { return nil, err } b.setNodeInfo(np) - // Tell it our membership: node-info reports the cluster principal on - // /v1/node-info but holds no cluster dir to read it from, so this push is the - // only source. It runs on every spawn, which also covers a supervised restart. - b.pushClusterIdentityToNodeInfo() b.pushPeerMetadata() // Register node-info's service so the daemon advertises ni= on _nvpair-node. // node-info binds the fixed :14318 (force_ports is inert), so the broker @@ -1394,10 +1383,8 @@ func (b *Broker) clusterManagerConfigDir() string { // configured, else nil. Threaded into every worker that does cluster-scoped // inter-node mTLS (nvpair-errors, nvpair-workload-manager, nvpair-node-scanner, // nvpair-manual-nodes, nvpair-engine-manager) so each serves/dials pinned cluster -// peers once this node is clustered; absent it, those workers stay on plain HTTP -// (or, for engine-manager's ec surface, don't bind it at all). nvpair-node-info is -// deliberately excluded — it stays plain HTTP even on a clustered node (see the -// spawnNodeInfo call site), so it is never passed --cluster-dir. +// peers once this node is clustered. Node-info also receives this directory for +// its peer node inventory handler, alongside independent public display HTTP. func (b *Broker) clusterDirArgs() []string { if b.clusterDir == "" { return nil @@ -1443,14 +1430,9 @@ func resolveLocalNodeID(clusterDir string) string { // the rest of its life — in the roster and ranked by the scheduler, yet // exchanging no cluster traffic. // -// Two workers are told directly, because they publish this node's membership -// rather than deriving it. The scanner owns the mDNS record: cluster-uuid= is how -// a peer finds the pin for this node, so it should flip now rather than at the -// scanner's next membership poll. node-info carries the same fact over plain -// HTTP, which is a peer's only way to learn it when this node's mDNS record does -// not reach it — and it holds no cluster dir, so the value has to be pushed. Both -// are idempotent and run on their own goroutine, because they are -// bounded-but-blocking writes and the caller is the cluster-manager's reader. +// Scanner updates its local announcement immediately. Node-info reads the shared +// cluster directory itself. The scanner write is asynchronous because this is +// cluster-manager's reader. func (b *Broker) applyClusterIdentityChange() { if b.clusterDir == "" { return @@ -1459,38 +1441,6 @@ func (b *Broker) applyClusterIdentityChange() { if sc := b.getScanner(); sc != nil { go sc.reloadIdentity() } - go b.pushClusterIdentityToNodeInfo() -} - -// clusterPrincipal is the cluster principal this node currently holds, or empty -// when it belongs to no cluster. It is read from the trust store on each call -// rather than cached: membership changes under the broker while it runs, and this -// is the value peers key their pins on. -func (b *Broker) clusterPrincipal() string { - if b.clusterDir == "" { - return "" - } - mesh := clustertrust.Open(b.clusterDir) - mesh.Refresh() - if !mesh.Clustered() { - return "" - } - return mesh.NodeUUID() -} - -// pushClusterIdentityToNodeInfo sends node-info the current cluster principal. -// Best-effort on a node-info that isn't running (never spawned, or mid-restart): -// the next spawn pushes again. A failed write is warned about rather than traced, -// because until the next push lands peers cannot learn this node's membership -// over HTTP — which is the whole point of reporting it. -func (b *Broker) pushClusterIdentityToNodeInfo() { - np := b.getNodeInfo() - if np == nil { - return - } - if err := np.SetClusterIdentity(b.clusterPrincipal()); err != nil { - slog.Warn("failed to push cluster identity to node-info", "err", err) - } } // forwardEngineNotification / forwardManualNodesNotification / @@ -2785,24 +2735,13 @@ func (b *Broker) forwardClusterManagerNotification(method string, params json.Ra } } -// applyClusterTrustChange re-derives the scanner's per-peer trust annotation -// after nvpair-cluster-manager reports a pin-set change. Fire-and-forget on its -// own goroutine, for the same reason as applyClusterIdentityChange: it is a -// bounded-but-blocking RPC and the caller is the cluster-manager's reader. -// -// node-info is pushed here too, not only from applyClusterIdentityChange: a pin -// on its own can decide membership (a cluster dir predating admission.json is -// clustered on pins alone — see clustertrust.Mesh), so this event can change the -// principal we publish without any identity change being reported. Peers now -// read that value over HTTP in preference to our mDNS record, so it must not have -// fewer refresh triggers than the record does — otherwise a peer converges on a -// principal we have already stopped advertising. +// applyClusterTrustChange asks scanner to rederive trust after the pin set changes. +// Node-info follows live membership from the same cluster directory. func (b *Broker) applyClusterTrustChange() { slog.Info("cluster pin set changed; re-deriving peer trust") if sc := b.getScanner(); sc != nil { go sc.reloadTrust() } - go b.pushClusterIdentityToNodeInfo() } // forwardWorkloadManagerNotification is the hook startWorkloadManager diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 67db946d9..510729f18 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -674,29 +674,3 @@ func writeSetLevelFrame(mu *sync.Mutex, w io.Writer, level string) error { _, err = w.Write(data) return err } - -// writeClusterIdentityFrame marshals a newline-delimited -// nodeinfo:set-cluster-identity notification and writes it to a child's stdin -// under mu. An empty principal is a real value ("this node is in no cluster"), -// so it is sent like any other. -func writeClusterIdentityFrame(mu *sync.Mutex, w io.Writer, clusterUUID string) error { - frame := struct { - JSONRPC string `json:"jsonrpc"` - Method string `json:"method"` - Params noderec.ClusterIdentityParams `json:"params"` - }{ - JSONRPC: "2.0", - Method: noderec.MethodSetClusterIdentity, - Params: noderec.ClusterIdentityParams{ClusterUUID: clusterUUID}, - } - data, err := json.Marshal(frame) - if err != nil { - return err - } - data = append(data, '\n') - - mu.Lock() - defer mu.Unlock() - _, err = w.Write(data) - return err -} diff --git a/services/nvpair-ui-broker/nodeinfo.go b/services/nvpair-ui-broker/nodeinfo.go index b0c1f6af0..629a65e64 100644 --- a/services/nvpair-ui-broker/nodeinfo.go +++ b/services/nvpair-ui-broker/nodeinfo.go @@ -31,7 +31,8 @@ const maxNodeInfoLine = 64 * 1024 // // The handle owns the OS process, the stdin pipe used to signal clean // shutdown (via EOF) and to forward runtime control frames -// (log/set-level), and a done channel that closes after cmd.Wait returns. +// (logging and peer metadata), and a done channel that closes after +// cmd.Wait returns. type nodeInfoProcess struct { cmd *exec.Cmd stdin io.WriteCloser @@ -49,16 +50,6 @@ func (n *nodeInfoProcess) SetLogLevel(level string) error { return writeSetLevelFrame(&n.stdinMu, n.stdin, level) } -// SetClusterIdentity tells node-info the cluster principal this node currently -// holds, so /v1/node-info can report it. node-info is spawned without a cluster -// dir (see spawnNodeInfo) and so cannot read membership for itself; this is the -// only thing that tells it. Sent on spawn and on every membership change, since -// an empty principal — a departure — is exactly the value peers need in order to -// stop treating this node as unavailable for pairing. -func (n *nodeInfoProcess) SetClusterIdentity(clusterUUID string) error { - return writeClusterIdentityFrame(&n.stdinMu, n.stdin, clusterUUID) -} - type localInfoObservation struct { info noderec.NodeInfo receivedAt time.Time diff --git a/services/nvpair-ui-broker/nodeinfo_identity_test.go b/services/nvpair-ui-broker/nodeinfo_identity_test.go deleted file mode 100644 index a5ca2e010..000000000 --- a/services/nvpair-ui-broker/nodeinfo_identity_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "strings" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "nvpair-shared/noderec" -) - -// TestWriteClusterIdentityFrame pins the wire form of the push node-info decodes. -// node-info reads its stdin as newline-delimited JSON-RPC, so a missing newline or -// a renamed method silently stops membership reaching it — and the only symptom -// would be peers going on suppressing an invite for a node that has left. -func TestWriteClusterIdentityFrame_EmitsNewlineTerminatedNotification(t *testing.T) { - test := func(name, clusterUUID string) { - t.Run(name, func(t *testing.T) { - var buf bytes.Buffer - var mu sync.Mutex - require.NoError(t, writeClusterIdentityFrame(&mu, &buf, clusterUUID), "write") - - line := buf.String() - assert.True(t, strings.HasSuffix(line, "\n"), "frame is not newline-terminated; node-info reads line-delimited frames") - assert.Equal(t, 1, strings.Count(line, "\n"), "frame must contain exactly one newline") - - var frame struct { - JSONRPC string `json:"jsonrpc"` - Method string `json:"method"` - ID json.RawMessage `json:"id"` - Params noderec.ClusterIdentityParams `json:"params"` - } - require.NoError(t, json.Unmarshal([]byte(line), &frame), "decode frame %q", line) - assert.Equal(t, "2.0", frame.JSONRPC) - assert.Equal(t, noderec.MethodSetClusterIdentity, frame.Method, "method") - // A notification, not a request: node-info's stdout is drained to - // io.Discard, so an id-bearing frame would strand a reply. - assert.Empty(t, frame.ID, "the push must be a notification") - assert.Equal(t, clusterUUID, frame.Params.ClusterUUID, "clusterUuid") - // The field must be on the wire even when empty, since that is how a - // departure is expressed. - assert.Contains(t, line, `"clusterUuid"`, "frame omitted clusterUuid") - }) - } - test("a principal", "our-principal") - // A departure is the value peers are waiting for, so it is sent like any - // other rather than skipped as empty. - test("a departure", "") -} diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index 0b67c63a0..b3f6601fb 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -461,6 +461,8 @@ type NodeInfo struct { MSSince int64 `json:"msSince"` HostUUID string `json:"hostUuid,omitempty"` ClusterUUID *string `json:"clusterUuid,omitempty"` + // Peer is included only in peer node inventory served over pinned mTLS. + Peer *PeerMetadata `json:"peer,omitempty"` } // Aged returns a copy whose sample age includes time spent in a queue or cache. diff --git a/services/shared/noderec/noderec_test.go b/services/shared/noderec/noderec_test.go index 45b6f0fde..e173d177b 100644 --- a/services/shared/noderec/noderec_test.go +++ b/services/shared/noderec/noderec_test.go @@ -35,6 +35,24 @@ func TestSampleAging_PreservesValidityAndFreshness(t *testing.T) { test("age saturates", NodeInfo{TelemetryValid: true, MSSince: 1<<63 - 2}, time.Second, 1<<63-1) } +func TestNodeInfo_PeerMetadataIsOptionalOnTheWire(t *testing.T) { + test := func(name string, peer *PeerMetadata) { + t.Run(name, func(t *testing.T) { + body, err := json.Marshal(NodeInfo{HostUUID: "host-a", Peer: peer}) + require.NoError(t, err) + var fields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(body, &fields)) + _, present := fields["peer"] + assert.Equal(t, peer != nil, present) + var decoded NodeInfo + require.NoError(t, json.Unmarshal(body, &decoded)) + assert.Equal(t, peer, decoded.Peer) + }) + } + test("public hardware", nil) + test("peer node inventory", &PeerMetadata{Name: "workstation", Addresses: []string{"192.0.2.20"}, Services: map[ServiceKey]int{ServiceNodeInfo: 14318}}) +} + func TestEngineModels_PrefersPerEngineInventoryAndFallsBackToLegacyModels(t *testing.T) { // Attribution present: each engine gets exactly its own list, never the union. attributed := DirectoryNode{ From 611c913a88bae860ba0856dae9820702d53c49d0 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Tue, 20 Oct 2026 09:29:06 -0400 Subject: [PATCH 4/8] Authenticate discovery candidates before promoting peer inventory Track unverified announcements by discoveryRecordId and promote a distinct canonical peer only after exact-pin mTLS and matching response identities. Take peer names, addresses, ports, and hardware from authenticated registry metadata. Scanner remote inventory, liveness, models, and scheduling samples use the verified target; failed candidates cannot overwrite or remove it. Keep candidate authentication separate from telemetry workers, preserve polling/backoff and sample age, and recheck current pins and local identity before accepting reused or in-flight reads. Remove public HTTP enrichment, operator-CA scanner flags, and unauthenticated ghost replacement. Retain the prior authenticated source link solely for correct expiry when an announcement update fails authentication. Discard queued and in-flight model results after any authenticated address-list change, including a removed secondary address. The per-node helper snapshots the complete list so unchanged multihomed peers retain failover and pass result validation. Suggested review order: - services/nvpair-node-scanner/spec.md as the reference contract, then docs/architecture.mdx - authentication, enrichment, and liveness boundaries. - services/shared/clustertrust/mesh_test.go, then mesh.go; peerclient_test.go, then peerclient.go - exact current certificate validation after reads, bounded clients, and redirect refusal. - services/shared/noderec/noderec_test.go, then noderec.go; services/shared/discovery/discovery_test.go, then discovery.go - value equality, absent-versus-empty inventory, trust contracts, and typed discovery events. - services/nvpair-node-scanner/directory_test.go, then directory.go - separate candidate keys and atomic promotion. - Scanner daemon_test.go, activity_test.go, cluster_membership_test.go, connreuse_test.go, node_info_outage_test.go, refresh_test.go, telemetry_test.go, and enrich_hosts_test.go - authenticated identity, expiry, age, concurrency, failed reads, and address changes. In daemon_test.go, include TestRefreshNodeModels_AcceptsPinnedPeersWithMultipleAddresses, TestDiscovery_DiscardsModelsAfterSecondaryAddressChanges, and TestDiscovery_DiscardsQueuedModelsForChangedAddresses. - Scanner daemon.go and node_info_gate.go - candidate authentication, canonical refreshes, accepted-result guards, and nonblocking candidate acquisition that protects verified sampling. Then enrich_hosts.go, main.go, and scanner.go. - Removed scanner tls.go, tls_test.go, ghost_node_test.go, and cluster_identity_refresh_test.go - obsolete public/CA and claimed-identity paths, with replacement coverage above. - services/tests/models_http_test.go and models_refresh_test.go - authenticated peer metadata and model inventory across real workers. - Scanner and broker READMEs, then desktop/docs/services-api.md - regenerated contract inventory. Validation: observed failing spoofing/promotion/pin-removal regressions before implementation, then failing size, address, local-ranking, and expiry cases before their fixes. Shared, scanner, node-info, and broker Go suites pass. The real-child scanner test and five controlled transport/scheduler integration tests pass. Desktop unit tests, typecheck, lint, dead-code, contracts, all 13 Windows binary builds, formatting, whitespace, and SPDX checks pass. Two existing Go tests and two desktop tests remain platform skips. Self-review fixes include isolated test identity storage and candidate linkage during failed updates. Signed-off-by: Kaylee Lubick --- desktop/docs/services-api.md | 2 +- docs/architecture.mdx | 76 +- services/nvpair-node-scanner/README.md | 67 +- services/nvpair-node-scanner/activity_test.go | 11 +- .../cluster_identity_refresh_test.go | 327 ---- .../cluster_membership_test.go | 100 +- .../nvpair-node-scanner/connreuse_test.go | 7 +- services/nvpair-node-scanner/daemon.go | 1403 +++++++--------- services/nvpair-node-scanner/daemon_test.go | 1485 +++++++++++++++-- services/nvpair-node-scanner/directory.go | 219 +-- .../nvpair-node-scanner/directory_test.go | 91 +- services/nvpair-node-scanner/discovery.go | 23 +- services/nvpair-node-scanner/enrich_hosts.go | 13 +- .../nvpair-node-scanner/enrich_hosts_test.go | 9 + .../nvpair-node-scanner/ghost_node_test.go | 422 ----- services/nvpair-node-scanner/main.go | 27 +- .../nvpair-node-scanner/node_info_gate.go | 33 +- .../node_info_outage_test.go | 119 -- services/nvpair-node-scanner/refresh_test.go | 166 +- services/nvpair-node-scanner/scanner.go | 13 +- services/nvpair-node-scanner/spec.md | 20 +- .../nvpair-node-scanner/telemetry_test.go | 679 ++------ services/nvpair-node-scanner/tls.go | 95 -- services/nvpair-node-scanner/tls_test.go | 93 -- services/nvpair-ui-broker/README.md | 8 +- services/shared/clustertrust/mesh.go | 13 +- services/shared/clustertrust/mesh_test.go | 23 + services/shared/clustertrust/peerclient.go | 26 +- .../shared/clustertrust/peerclient_test.go | 11 +- services/shared/discovery/discovery.go | 11 +- services/shared/discovery/discovery_test.go | 6 +- services/shared/noderec/noderec.go | 73 +- services/shared/noderec/noderec_test.go | 58 + services/tests/models_http_test.go | 113 +- services/tests/models_refresh_test.go | 71 +- 35 files changed, 2677 insertions(+), 3236 deletions(-) delete mode 100644 services/nvpair-node-scanner/cluster_identity_refresh_test.go delete mode 100644 services/nvpair-node-scanner/ghost_node_test.go delete mode 100644 services/nvpair-node-scanner/tls.go delete mode 100644 services/nvpair-node-scanner/tls_test.go diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index 8137f326d..da69f9353 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -178,7 +178,7 @@ | `ready` | notification (we consume) | ✅ yes | **Dynamic / unresolved notify sites (verify by hand — `npm run service-contracts` prints the line numbers):** -- `method (var) (daemon.go, 3 sites)` +- `method (var) (daemon.go, 4 sites)` ## nvpair-node-settings diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 0dc74e872..3981ef5d0 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -98,7 +98,7 @@ the layers that are checked: | Engines | A readiness probe on start, then a periodic health probe. Status carries a `healthy` flag, and a failed probe raises a service error. | | Workers | Process liveness only. An exit is detected and restarted. A hang is not detected. | | The broker | A `ping` request clients can call, which the terminal interface's **Service** tab uses. | -| Peer nodes | Reachability, not health: missed announcements, then recent inference activity, recent telemetry, or a probe of the node-info or engine-manager port before eviction. The inference proxies report activity but are never probed. | +| Peer nodes | Reachability, not health: missed announcements are checked against recent authenticated node-info samples and inference activity. The inference proxies report activity but are never probed. | Worker supervision is restart-based. An unexpected exit is retried with exponential backoff from roughly one second to 16, with a budget of five attempts @@ -249,13 +249,23 @@ fresh. Scanner's owned local registry supplies node-info's peer metadata over th reverse IPC path; discovered announcements cannot supply local measurements or metadata. +Scanner keeps unverified announcements under separate discovery record IDs. +It promotes a candidate only after reading peer node inventory over exact-pin +mTLS and checking both response identities against that certificate. The peer's +name, addresses, service ports, and hardware then come from the authenticated +response. Discovery claims and public HTTP cannot replace that canonical peer +record. Candidate authentication has separate worker permits from verified +telemetry polling. Scanner rechecks live pins before accepting results, including +results returned over reused connections. + Electron polls every discovered node's `/v1/node-info` directly because that detail is richer than the broker's control surface exposes. Healthy nodes are sampled every two seconds; consecutive failures back off to 4, 8, 16, then 30 seconds and reset on success. This is the one place Electron makes service-data -HTTP calls of its own. Separately, the backend scanner samples a compact -maximum-GPU utilization value for scheduling with the same healthy cadence and -capped failure backoff, which the broker relays straight to the job scheduler. +HTTP calls of its own. Separately, the backend scanner reads each verified peer's +complete node-info response over pinned mTLS with the same healthy cadence and +capped failure backoff. Each accepted response updates inventory and supplies +compact maximum-GPU telemetry, which the broker relays to the job scheduler. Routing never depends on renderer metrics. ## Inference Data Flow @@ -627,58 +637,44 @@ long-lived receiver. Announcements supply discovery candidates and advertised service ports. Scanner enriches node records with details from two services: -- GPU, CPU, and memory inventory from node-info, currently over public HTTP for - remote nodes and private worker IPC for the local host +- GPU, CPU, and memory inventory from node-info, over pinned mutual TLS for + verified remote peers and private worker IPC for the local host - Its model list from engine-manager, over pinned mutual TLS for remote peers or loopback HTTP for the local host, as a flat union, a per-engine breakdown, and the per-engine set of models currently loaded in memory -The peer node inventory handler is available, but migrating remote hardware -reads to pinned mutual TLS and promoting verified records remain pending work -in the [scanner specification](../services/nvpair-node-scanner/spec.md). +Authenticated node-info supplies the canonical peer's name, addresses, and +registered ports as well as hardware. Scanner replaces only the successful +discovery with this verified record. Other observations remain independently +unverified. Consumer merge and replay enforcement remain subsequent work in the +[scanner specification](../services/nvpair-node-scanner/spec.md). Each enrichment keeps a last-good value, so one failed fetch dims nothing. A node's card does not blank out because a single poll timed out. -With a cluster directory configured, a browsed peer whose `cluster-uuid=` matches -a pin this node holds is annotated as trusted, which is how the interface can -distinguish a paired node from a stranger it can merely see. This annotation -matches an advertised identity to a saved pin; it does not authenticate the -announcing endpoint. +`trusted` requires a completed peer read against the current exact pin. A pin +addition schedules authentication; it never makes an announcement trusted. ### Node Eviction The scanner scans every five seconds. Once a node has missed three scans in a row, about fifteen seconds, the scanner checks on it at every scan, and any answer resets the count. It drops the node only after twelve consecutive misses, about a -minute, with every check along the way failing. These are the checks: +minute, with every check along the way failing. For a promoted peer the checks are: - Inference the node served in the last minute, which the proxies report -- Recent telemetry from that node -- A direct request to the node's node-info endpoint -- A probe of the engine-manager port - -The activity check comes first because a node under load is the one most likely -to miss an announcement, and evicting it would pull a working machine out of the -routing pool at its busiest. - -The remaining checks deliberately avoid the inference proxy ports. Those ports -exist to serve requests, and using them as a liveness test would put connection -attempts on the serving path every time multicast dropped a packet. - -Multicast is lossy, and Wi-Fi, VPN adapters, and sleeping network interfaces all -produce gaps that have nothing to do with a node's health. Without that probe a -momentary gap would evict a perfectly healthy node and take its models out of the -routing pool. - -Discovery holds record replacement to the same standard. A node can reappear with -a new identity, for example after a reset. The old entry then looks superseded, -and leaving both in place would list the machine twice. But a matching address -and hostname arrive in an unauthenticated announcement that anything on the -network can send, so those signals only mark the old record as a suspect. -Discovery confirms the node's identity before dropping that record. A stale -duplicate therefore cannot linger, and nothing can displace a peer by claiming -its address. +- Authenticated inventory from that node in the last ten seconds + +These checks use accepted evidence without starting another network request. +The canonical sampling loop continues independently, so multicast loss does not +add synchronous probes to discovery processing. Recent inventory or inference +activity keeps a healthy node in the routing pool through Wi-Fi, VPN, and sleep +or wake gaps. Inference proxy ports are never probed for liveness. + +A reset node may reappear under a new identity. Its unverified announcement +cannot replace a verified record by matching its name or address. The old +record follows authenticated liveness and trust invalidation. Public HTTP +identity reports and open TCP ports cannot remove or retain verified peers. ### Manual Nodes diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index fb473ad1f..f8f2b7e22 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -10,20 +10,56 @@ SPDX-License-Identifier: Apache-2.0 Concretely, the daemon: - **Advertises** this node's one `_nvpair-node._tcp` record, built from a registry of the local services its parent (the broker) registers — `ni`/`ol`/`lm`/`er`/`wl`/`cl`/`em`/`ec` ports plus the node identity (`uuid=`, `cluster-uuid=` when clustered, `ip=`). One record per node covers every service. -- **Browses** `_nvpair-node._tcp` for every node on the LAN (including itself) and maintains a queryable directory keyed by host UUID. -- **Enriches** each discovered node: GPU/CPU/memory inventory from its `ni` (node-info) port over plain HTTP, and the model list from its `em` (engine-manager) port (`GET /v1/models`) over cluster mTLS — a peer's model inventory is cluster data, so it is fetched only when this node holds a pin for the principal that peer advertises, while this node's own list is read over loopback in plaintext — the flat union, the per-engine breakdown (`modelsByEngine`), and the per-engine set of models loaded in memory (`loadedByEngine`), all carried onto the node's directory entry so remote cards can show loaded state. Each enrichment has a last-good cache so a transient fetch miss doesn't blank the node's card. Node-info enrichment can optionally be moved onto HTTPS (see the TLS flags below) — off by default and gated only on operator flags, never inferred per-node. +- **Browses** `_nvpair-node._tcp` and tracks connection candidates by + `discoveryRecordId`. Claimed UUIDs, names, and addresses cannot select a local + or verified peer record. +- **Authenticates peers** using exact certificate pins from + `nvpair-shared/clustertrust`. A successful pinned mTLS node-info read must + report matching host and certificate UUIDs and complete peer metadata. + The authenticated name, addresses, ports, and hardware form a distinct + `pinned` peer record, replacing only that successful discovery. Failed + candidates stay separate and `unverified`. Public HTTP is never a scanner + measurement source, and operator CAs cannot substitute for cluster pins. +- **Refreshes models** every 15 seconds from verified endpoints, preserving + the last successful inventory during outages. A newly authenticated peer + schedules a model refresh immediately. Remote model lists use pinned + mTLS at the authenticated engine-manager port. Local models use the owned + engine-manager over loopback; local hardware comes from private worker IPC. + Local registry publications use retained models and wake the model sweep, + so an HTTP read cannot hold the lock needed by owned hardware notifications. - **Samples scheduling telemetry** from healthy nodes every two seconds with stable per-node jitter. Consecutive remote failures back off to 4, 8, 16, then 30 seconds and reset on success. Local inventory and samples arrive every two seconds from the owned node-info worker over private broker/worker IPC. - It emits only host UUID, maximum GPU utilization, validity, and - source-reported sample age. Remote telemetry reads do not mutate the public - directory; local IPC also updates changed hardware inventory. -- **Annotates trust:** with `--cluster-dir`, a browsed peer whose `cluster-uuid=` this node holds a pin for is marked `trusted` (via `nvpair-shared/clustertrust`). The annotation is re-derived on `discovery:reload-trust`, which the broker sends when `nvpair-cluster-manager` reports a pin-set change, as well as when a peer's mDNS record changes. Both triggers are needed: a peer that was already advertising when this node joined never moves its record again, so the browse path alone would leave it annotated from before this node's own pins existed. A change emits one `discovery:node-updated`; a reload that finds nothing to change is silent. + Each accepted peer read updates directory metadata and hardware and emits + compact scheduling telemetry and a full authenticated hardware snapshot. + Both outputs use the same sample with its elapsed age. Local IPC also updates + changed hardware inventory. +- **Follows live trust:** a pin addition schedules authentication. Pin removal, + replacement, or local departure removes the verified row and its scanner + caches and drops affected idle connections. Requests recheck the current pin + and serving identities before accepting results, including reused connections + and reads already in progress. - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. -- **Anti-flap:** a node has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. +- **Retains verified peers during multicast gaps:** the browser retains its + twelve missed five-second scans and six empty-scan grace. A promoted peer's + authenticated inventory from the last 10 seconds, or approved inference + activity from the last 60 seconds, supplies liveness evidence. Canonical reads + continue on the sampling loop, without separate synchronous liveness probes. + An open TCP port or another discovery claiming that UUID cannot keep the peer alive. + A peer retained after its last discovery expires is still checked during + the sampling loop. A minute without authenticated inventory or inference + activity removes its scanner record and caches, preserving membership and pins. - **A scan that loses everything is treated as our own fault:** when more than one node is known and a scan returns *none* of them, no node is penalized for it — for up to six consecutive scans (~30s), after which they count normally. Independent machines do not leave the network in the same five-second window, but a process too starved to drain its multicast socket within `scanTimeout` hears silence from all of them at once; without this, one saturated machine dropped its entire cluster, idle peers included, in a single reconcile. The suppression needs two or more known nodes to engage: losing a lone peer looks the same whether it left or we stopped listening, so that case falls through to the ordinary threshold. -- **Converges without mDNS:** a 15s sweep currently re-reads each peer over public HTTP and folds changes into the directory, independent of browse events. This covers missed announcements and facts that change without moving the mDNS record. `clusterUuid` remains available on public node-info, including when its separate peer node inventory handler is enabled. Membership is refreshed before models, because model reads use pinned mTLS keyed on that principal. Failed or absent membership reports leave the annotation unchanged. Migrating these remote inventory reads to authenticated promotion is the next implementation step in [the spec](spec.md). +- **Separates pending authentication from telemetry:** candidates have their own + bounded workers and retry state. Slow candidates cannot occupy verified-peer + telemetry workers or replace their polling targets. Authentication and + telemetry failures use the existing capped backoff. Requests retain deadlines, + response-size limits, address failover, and one connection per node-info + origin. Requests never follow redirects to another endpoint or transport. + +Queued and in-flight model results are discarded if the peer's authenticated +address list changes, including a secondary address used for failover. ## Communication @@ -34,10 +70,7 @@ Bidirectional newline-delimited JSON-RPC 2.0. Stdio by default; `--ipc ` s | Flag | Default | Description | |---|---|---| | `--ipc ` | _(stdio)_ | IPC endpoint: Unix socket or Windows named pipe | -| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`). Gates four things: a browsed peer holding a pin we trust is annotated `trusted`; this node advertises `cluster-uuid=` while it is actually a member; a peer's `em` model list is fetched over cluster mTLS; and the directory above it is the base this node's identity is resolved from | -| `--ca-bundle ` | _(none)_ | PEM CA bundle to trust for verifying node-info server certs (additive to the system trust store). Setting any TLS flag moves node-info enrichment onto HTTPS | -| `--client-cert ` | _(none)_ | PEM client certificate to present when fetching `/v1/node-info` from TLS-enabled nodes (mTLS; requires `--client-key`) | -| `--client-key ` | _(none)_ | PEM client private key matching `--client-cert` | +| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`) for pinned peer authentication and local identity | | `--log-level ` | `info` | `error` / `warn` / `info` / `debug` | | `--version` | | Print version and exit | @@ -52,7 +85,7 @@ Bidirectional newline-delimited JSON-RPC 2.0. Stdio by default; `--ipc ` s | `discovery:update-txt` | `{service, port, txt?}` | `{ok:true}` — same as register with a new TXT | | `discovery:get-nodes` | `{service?}` | `{nodes:[DirectoryNode]}` — the directory, optionally filtered to nodes advertising one service | | `discovery:reload-identity` | — | `{ok:true}` — re-resolve and re-advertise this node's identity | -| `discovery:reload-trust` | — | `{ok:true}` — re-derive `trusted` on every directory entry from the current pin set | +| `discovery:reload-trust` | — | `{ok:true}` — invalidate stale peer proofs and schedule authentication against current pins | | `discovery:set-observed-addresses` | `{addresses}` | `{ok:true}` — the local addresses peers have actually reached this node on, so address selection can rank a peer-proven address above an inferred one | | `shutdown` | — | `null` | @@ -81,10 +114,10 @@ Emitted as the directory changes. Each carries the affected `DirectoryNode` (ide Records also carry `discoveryRecordId` and `identityVerification`. The local registry record is `local`; browsed records are `unverified`, including when their claimed -principal has a pin. `pinned` denotes authenticated identity in the shared -contract. Discovery record IDs identify connection candidates independently of UUID -and principal claims. Their address order does not affect the ID. The existing -`trusted` pin annotation alone does not prove endpoint ownership. +principal has a pin. A successful read creates a distinct `peer:` record +with `pinned` verification and `trusted: true`. Failed discoveries cannot alter +that record. Discovery record IDs identify candidates independently of UUID and +principal claims; address order does not affect the ID. - `discovery:node-discovered` — a new node entered the directory - `discovery:node-updated` — a known node's record or enrichment changed. A re-observation that changes nothing emits no event, so `lastSeen` is a last-*change* stamp rather than a liveness heartbeat diff --git a/services/nvpair-node-scanner/activity_test.go b/services/nvpair-node-scanner/activity_test.go index d804310f2..bf3301fff 100644 --- a/services/nvpair-node-scanner/activity_test.go +++ b/services/nvpair-node-scanner/activity_test.go @@ -44,8 +44,11 @@ func unreachableNode(t *testing.T, hostUUID string) RawNode { // is streaming a generation back to us is evicted for failing a probe it was too // busy to answer. func TestFreshActivity_KeepsANodeNoProbeCanReach(t *testing.T) { - d := activityDaemon() - n := unreachableNode(t, "busy-node") + f := newDiscoveryPeerFixture(t, "busy-node") + f.discover(t) + d, n := f.daemon, f.raw + f.server.Close() + delete(d.lastInfoAt, "busy-node") require.False(t, d.reachable(n), "precondition: a node with no listener must fail the probe when nothing vouches for it") @@ -130,11 +133,7 @@ func TestSelfActivity_IsDropped(t *testing.T) { // report vouch for the node the moment it was rediscovered. func TestForget_ClearsActivity(t *testing.T) { d := activityDaemon() - d.lastInfo = make(map[string]NodeInfoResponse) d.lastInfoAt = make(map[string]time.Time) - d.lastModels = make(map[string][]string) - d.lastModelsByEngine = make(map[string]map[string][]string) - d.lastLoadedByEngine = make(map[string]map[string][]string) d.nodeInfoDown = make(map[string]bool) d.noteActivity("node", 0) diff --git a/services/nvpair-node-scanner/cluster_identity_refresh_test.go b/services/nvpair-node-scanner/cluster_identity_refresh_test.go deleted file mode 100644 index d29c6d136..000000000 --- a/services/nvpair-node-scanner/cluster_identity_refresh_test.go +++ /dev/null @@ -1,327 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "path/filepath" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "nvpair-shared/clustertrust" - "nvpair-shared/clustertrusttest" - "nvpair-shared/noderec" -) - -// The mDNS-independent cluster-membership half of the periodic refresh loop -// (daemon.refreshClusterIdentityOnce). These tests drive it against a stub -// /v1/node-info server, with no mDNS anywhere: the whole point is that a peer's -// membership converges without a browse event ever firing. - -// nodeInfoStub is a swappable /v1/node-info handler. `body` is the raw JSON so a -// test can control exactly what the peer reports — including omitting clusterUuid -// entirely, which is how a node too old to know about the field answers. -type nodeInfoStub struct { - mu sync.Mutex - status int - body string -} - -func (s *nodeInfoStub) set(status int, body string) { - s.mu.Lock() - s.status = status - s.body = body - s.mu.Unlock() -} - -func (s *nodeInfoStub) handler(t *testing.T) http.HandlerFunc { - t.Helper() - return func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - http.NotFound(w, r) - return - } - s.mu.Lock() - status, body := s.status, s.body - s.mu.Unlock() - if status == 0 { - status = http.StatusOK - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _, err := w.Write([]byte(body)) - assert.NoError(t, err) - } -} - -func startNodeInfoStub(t *testing.T, body string) (*nodeInfoStub, int) { - t.Helper() - stub := &nodeInfoStub{} - stub.set(http.StatusOK, body) - srv := httptest.NewServer(stub.handler(t)) - t.Cleanup(srv.Close) - return stub, portFromURL(t, srv.URL) -} - -// newIdentityRefreshTestDaemon builds a daemon wired for the node-info refresh -// path only: a real directory and plain node-info client, an unclustered mesh (so -// it holds no pins, like a machine that has left), and no codec (emit is -// nil-safe). The registry self uuid is one no peer test seeds, so every seeded -// node is treated as a peer. -func newIdentityRefreshTestDaemon() *daemon { - return &daemon{ - reg: newRegistry("self-node-uuid", "", selfAddrs("127.0.0.1")), - dir: newDirectory(), - mesh: clustertrust.Open(""), - http: &http.Client{Timeout: nodeInfoFetchTimeout}, - lastInfo: make(map[string]NodeInfoResponse), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), - } -} - -// seedPeer puts a peer in the directory advertising node-info at port, annotated -// as belonging to clusterUUID — the state a browse event would have left behind. -func seedPeer(d *daemon, hostUUID, clusterUUID string, niPort int) { - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: hostUUID, - Name: hostUUID, - IP: "127.0.0.1", - ClusterUUID: clusterUUID, - Trusted: clusterUUID != "", - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: niPort}, - }, - }) -} - -// TestDepartedPeerConvergesWithoutMDNS is the regression guard for the reported -// failure: after every peer left the cluster, one machine went on showing them as -// belonging to another cluster with invites disabled, while the rest of the fleet -// showed them as available. -// -// Membership reaches this daemon as the cluster-uuid= key on a peer's mDNS record, -// read only when a browse *change* event fires. The peer left and re-advertised, -// but this host did not receive it — and since the peer was still reachable, the -// liveness probe kept the frozen entry alive rather than letting it be -// rediscovered. Nothing asked again, so the principal outlived the cluster. -// -// No browse event occurs anywhere in this test. The peer's own HTTP report has to -// be enough. -func TestDepartedPeer_ConvergesWithoutMDNS(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - // The peer has left, so it now reports no cluster principal... - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-uuid","clusterUuid":""}`) - // ...but our entry still carries the one it advertised before it left. - seedPeer(d, peerUUID, "their-old-principal", niPort) - - d.refreshClusterIdentityOnce() - - n, ok := d.dir.get(peerUUID) - require.True(t, ok, "peer dropped from the directory by the refresh") - assert.False(t, n.Clustered(), "peer still annotated as clustered (clusterUuid") - assert.False(t, n.Trusted, "peer still annotated as trusted after leaving the cluster") - // A second pass agrees with the first and stays silent. - assert.False(t, d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort), "refresh emitted again for an unchanged peer") -} - -func TestClusterIdentityRefresh_FallsBackToASecondPublishedAddress(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-multihomed" - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-multihomed","clusterUuid":""}`) - seedPeer(d, peerUUID, "stale-principal", niPort) - - require.True(t, d.refreshNodeClusterIdentityCandidates(peerUUID, []string{"127.0.0.2", "127.0.0.1"}, niPort), "membership did not refresh through the second published address") - n, ok := d.dir.get(peerUUID) - require.True(t, ok, "peer after refresh (%v)", n) - require.False(t, n.Clustered(), "peer after refresh (%v)", n) -} - -// TestPeerInForeignClusterStaysSuppressed is the other half of the contract, and -// the reason the refresh trusts only an explicit report: a peer that is genuinely -// in a cluster we are not part of must keep its annotation, so the invite stays -// suppressed rather than being offered and rejected. -func TestPeerInForeignCluster_StaysSuppressed(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-uuid","clusterUuid":"their-principal"}`) - seedPeer(d, peerUUID, "", niPort) - - d.refreshClusterIdentityOnce() - - n, _ := d.dir.get(peerUUID) - require.True(t, n.Clustered(), "peer reporting a cluster principal was not annotated as clustered") - assert.Equal(t, "their-principal", n.ClusterUUID) - // We hold no pin for it, so it is clustered but not trusted. - assert.False(t, n.Trusted, "peer in a cluster we hold no pin for was annotated trusted") -} - -// TestPeerWithoutClusterFieldIsLeftAlone pins the version-skew rule. A node too -// old to report clusterUuid says nothing about its membership, and silence must -// not be read as "unclustered" — doing so would mark a clustered peer invitable, -// the exact failure the annotation prevents. -func TestPeerWithoutClusterField_IsLeftAlone(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-uuid"}`) - seedPeer(d, peerUUID, "their-principal", niPort) - - assert.False(t, d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort), "refresh acted on a peer that reported no clusterUuid field") - n, _ := d.dir.get(peerUUID) - assert.Equal(t, "their-principal", n.ClusterUUID) -} - -// TestUnreachablePeerKeepsItsAnnotation covers the same rule for a failed fetch: -// no answer is not an answer. -func TestUnreachablePeer_KeepsItsAnnotation(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - stub, niPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":""}`) - stub.set(http.StatusInternalServerError, "") - seedPeer(d, peerUUID, "their-principal", niPort) - - assert.False(t, d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort), "refresh acted on a failed node-info fetch") - n, _ := d.dir.get(peerUUID) - assert.Equal(t, "their-principal", n.ClusterUUID) -} - -// TestSelfExcludedFromIdentityRefresh keeps the sweep off this node's own entry: -// publishSelf owns the local principal from the registry, and a loopback HTTP read -// must not become a second source for it. -func TestClusterIdentityRefresh_SkipsLocalNode(t *testing.T) { - d := newIdentityRefreshTestDaemon() - self := d.reg.record().HostUUID - - // Self's node-info would report unclustered, but the registry says otherwise. - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":""}`) - seedPeer(d, self, "our-principal", niPort) - - d.refreshClusterIdentityOnce() - - n, _ := d.dir.get(self) - assert.Equal(t, "our-principal", n.ClusterUUID, "self clusterUuid") -} - -// TestStrangerAtTheAddressIsNotApplied covers the case where the machine -// answering is not the one the entry describes — a wiped and reinstalled node, or -// a reused DHCP lease. node-info names who it really is, so its membership must -// not be written onto the previous occupant's entry. That entry ages out through -// the liveness probe's identity check instead, which owns the decision. -func TestStrangerAtTheAddress_IsNotApplied(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - // A different machine now answers here, and it belongs to no cluster. - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"someone-else","clusterUuid":""}`) - seedPeer(d, peerUUID, "their-principal", niPort) - - assert.False(t, d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort), "applied membership reported by a different host") - n, _ := d.dir.get(peerUUID) - assert.Equal(t, "their-principal", n.ClusterUUID) - - // A peer that reports no hostUuid predates the field and is taken at its word. - _, oldPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":""}`) - seedPeer(d, "old-peer-uuid", "their-principal", oldPort) - assert.True(t, d.refreshNodeClusterIdentity("old-peer-uuid", "127.0.0.1", oldPort), "skipped a peer that reported no hostUuid; it cannot be identity-checked either way") -} - -// TestSweepAnnotatesEveryPeerAgainstThePinSet exercises the sweep the way it -// actually runs: several peers at once, against a mesh holding a real pin. It -// covers the three answers a peer can give in one pass, and puts concurrent load -// on the fan-out so -race has something to observe. -func TestSweep_AnnotatesEveryPeerAgainstThePinSet(t *testing.T) { - clusterDir := filepath.Join(t.TempDir(), "cluster") - // This node is in a cluster and holds a pin for one of the peers below. - clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", "pinned-principal") - - d := newIdentityRefreshTestDaemon() - d.mesh = clustertrust.Open(clusterDir) - - // A fellow member, a machine in someone else's cluster, and one that has left. - for _, peer := range []struct { - uuid string - reported string - }{ - {"member-uuid", "pinned-principal"}, - {"stranger-uuid", "foreign-principal"}, - {"departed-uuid", ""}, - } { - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":"`+peer.reported+`"}`) - // Seed each one annotated wrongly, so every case has to be corrected. - seedPeer(d, peer.uuid, "stale-principal", niPort) - } - - d.refreshClusterIdentityOnce() - - member, _ := d.dir.get("member-uuid") - assert.Equal(t, "pinned-principal", member.ClusterUUID, "member = (clusterUuid") - assert.True(t, member.Trusted, "member = (clusterUuid") - stranger, _ := d.dir.get("stranger-uuid") - assert.Equal(t, "foreign-principal", stranger.ClusterUUID, "stranger = (clusterUuid") - assert.False(t, stranger.Trusted, "stranger = (clusterUuid") - departed, _ := d.dir.get("departed-uuid") - assert.False(t, departed.Clustered(), "departed = (clusterUuid") - assert.False(t, departed.Trusted, "departed = (clusterUuid") -} - -// TestPinSetPassCannotWriteBackAPrincipal guards the interaction between the two -// membership passes. Both write the same two fields and they run concurrently: the -// pin-set pass on the broker's reload-trust announcement, the sweep on its own -// tick. The pin-set pass knows the pin set but not what any peer advertises, so it -// supplies no principal and trust is derived from whatever is stored when its write -// lands. Were it to pass a principal read earlier — from the snapshot it iterates — -// a sweep landing in between would be silently reverted and the invite gate would -// flip in the UI. -// -// Asserted at the write path rather than by racing the two passes, so it fails -// deterministically: a nil principal must leave the stored one alone, and the trust -// it derives must come from that stored value. -func TestPinSetPass_CannotWriteBackAPrincipal(t *testing.T) { - dir := newDirectory() - dir.upsert(noderec.DirectoryNode{HostUUID: "peer", ClusterUUID: "current-principal"}) - - // Stands in for the sweep having just moved the principal: the pin-set pass - // holds a pin for the current value and none for what it used to be. - trustFor := func(clusterUUID string) bool { return clusterUUID == "current-principal" } - - n, changed := dir.applyClusterIdentity("peer", nil, trustFor) - require.True(t, changed, "pin-set pass reported no change, want the trust annotation derived") - assert.Equal(t, "current-principal", n.ClusterUUID) - assert.True(t, n.Trusted, "trust was not derived from the stored principal") - - // And a pass that does know the principal still governs it. - moved := "moved-principal" - n, _ = dir.applyClusterIdentity("peer", &moved, trustFor) - assert.Equal(t, "moved-principal", n.ClusterUUID, "after an explicit move = (clusterUuid") - assert.False(t, n.Trusted, "after an explicit move = (clusterUuid") -} - -// TestNodeInfoResponseDistinguishesAbsentFromEmpty pins the wire contract the -// version-skew rule depends on: the pointer has to survive decoding so absent and -// present-but-empty stay distinguishable. -func TestNodeInfoResponse_DistinguishesAbsentFromEmpty(t *testing.T) { - var absent NodeInfoResponse - require.NoError(t, json.Unmarshal([]byte(`{"GPUs":[]}`), &absent), "decode absent") - assert.Nil(t, absent.ClusterUUID, "absent clusterUuid decoded to") - assert.False(t, absent.TelemetryValid, "absent telemetry fields decoded as valid (%v)", absent) - assert.Equal(t, int64(0), absent.MSSince, "absent telemetry fields decoded as valid (%v)", absent) - - var empty NodeInfoResponse - require.NoError(t, json.Unmarshal([]byte(`{"GPUs":[],"clusterUuid":"","telemetryValid":true,"msSince":137}`), &empty), "decode empty") - require.NotNil(t, empty.ClusterUUID, "present-but-empty clusterUuid decoded to nil, want a non-nil empty string") - assert.Equal(t, "", *empty.ClusterUUID, "empty clusterUuid") - assert.True(t, empty.TelemetryValid, "telemetry fields = valid") - assert.Equal(t, int64(137), empty.MSSince, "telemetry fields = valid") -} diff --git a/services/nvpair-node-scanner/cluster_membership_test.go b/services/nvpair-node-scanner/cluster_membership_test.go index f951ac4f7..f7105a257 100644 --- a/services/nvpair-node-scanner/cluster_membership_test.go +++ b/services/nvpair-node-scanner/cluster_membership_test.go @@ -14,7 +14,6 @@ import ( "nvpair-shared/clustertrust" "nvpair-shared/clustertrusttest" - "nvpair-shared/noderec" ) func newMembershipTestDaemon(clusterDir string) *daemon { @@ -26,7 +25,6 @@ func newMembershipTestDaemon(clusterDir string) *daemon { baseDir: filepath.Dir(clusterDir), reg: newRegistry("host-uuid", "", selfAddrs("127.0.0.1")), dir: newDirectory(), - lastInfo: make(map[string]NodeInfoResponse), lastInfoAt: make(map[string]time.Time), lastModels: make(map[string][]string), lastModelsByEngine: make(map[string]map[string][]string), @@ -65,9 +63,9 @@ func TestModelFetchTransport_FollowsMembership(t *testing.T) { _, _, ok = d.modelsClient("192.168.1.99", "principal-stranger") require.False(t, ok, "an unpinned peer must not be fetched") // Self keeps its loopback plaintext path once clustered too. - _, scheme, ok = d.modelsClient("127.0.0.1", "principal-self") - require.True(t, ok, "clustered loopback: scheme (%v, %v)", scheme, ok) - require.Equal(t, "http", scheme, "clustered loopback: scheme (%v, %v)", scheme, ok) + _, scheme, ok = d.modelsClient("127.0.0.1", "") + require.True(t, ok) + assert.Equal(t, "http", scheme) } // TestClusterUUIDGatedOnLiveMembership pins the invite-deadlock fix at the @@ -114,76 +112,6 @@ func TestClusterUUID_AppearsOnJoinWithoutRestart(t *testing.T) { require.Equal(t, "principal-C", d.reg.record().ClusterUUID, "post-join cluster-uuid") } -// peerEntry is a directory entry for a peer already advertising a cluster -// principal — the shape a browse event produces for a node that is a member of a -// cluster we may or may not have joined yet. -func peerEntry(hostUUID, clusterUUID string) noderec.DirectoryNode { - return noderec.DirectoryNode{ - HostUUID: hostUUID, - Name: hostUUID, - IP: "192.0.2.10", - ClusterUUID: clusterUUID, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, - } -} - -// TestReloadTrustAdoptsPeerPinnedAfterDiscovery is the regression guard for the -// reported failure, driven through the RPC entry point the broker calls. -// -// A peer is discovered while this node is still unclustered, so it is annotated -// untrusted — correctly, at that instant. The join then lands, but the peer's -// mDNS record never changes again, so no further browse event is produced and -// nothing re-asks the question. That entry stayed untrusted for the life of the -// process: both proxies dropped the peer as an unpinned relay peer, so the node -// listed only its own models and never routed inference anywhere, while its -// roster and pins on disk said it was a healthy member of a three-node cluster. -func TestReloadTrust_AdoptsPeerPinnedAfterDiscovery(t *testing.T) { - const peerUUID = "principal-peer" - clusterDir := filepath.Join(t.TempDir(), "cluster") - d := newMembershipTestDaemon(clusterDir) - - // Discovered before we hold any cluster identity. - d.dir.upsert(toDirectoryNodeTrusted(peerEntry("peer-host", peerUUID), d)) - n, _ := d.dir.get("peer-host") - require.False(t, n.Trusted, "peer annotated trusted while this node held no pins") - - // The join lands on disk and the cluster-manager announces it. No browse - // event accompanies it, and none ever will. - clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) - d.reloadTrust() - - n, _ = d.dir.get("peer-host") - require.True(t, n.Trusted, "peer still untrusted after its pin landed; routing would drop it") - // The same pass converges what we advertise, so peers can pin us back. - require.Equal(t, "principal-self", d.reg.record().ClusterUUID, "advertised cluster-uuid") - - // A removal is picked up just as promptly, and for the same reason. - clustertrusttest.RemovePeerPin(t, clusterDir, peerUUID) - d.reloadTrust() - - n, _ = d.dir.get("peer-host") - require.False(t, n.Trusted, "peer still trusted after its pin was removed") -} - -// TestTrustReconcileIsSilentWhenNothingChanged keeps the reconcile from becoming -// an event source of its own: it runs every couple of seconds against every -// known node, so an unconditional write would push a directory-sized -// discovery:nodes snapshot through the broker to every subscriber on each pass. -func TestTrustReconcile_IsSilentWhenNothingChanged(t *testing.T) { - const peerUUID = "principal-peer" - clusterDir := filepath.Join(t.TempDir(), "cluster") - clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) - d := newMembershipTestDaemon(clusterDir) - - d.dir.upsert(toDirectoryNodeTrusted(peerEntry("peer-host", peerUUID), d)) - d.dir.upsert(toDirectoryNodeTrusted(peerEntry("stranger", "principal-stranger"), d)) - - for _, hostUUID := range []string{"peer-host", "stranger"} { - _, changed := d.dir.applyClusterIdentity(hostUUID, nil, d.mesh.HasPin) - require.False(t, changed, " (%v)", hostUUID) - } -} - // TestReconcileIdentityFollowsMembership covers the identity half of the same // pass: it re-advertises only when live membership disagrees with what the // registry is publishing, so the common case costs a string compare. @@ -218,30 +146,16 @@ func TestPeerLeavingItsCluster_ClearsInviteGate(t *testing.T) { ID: "peer", TXT: []string{"v=1", "uuid=" + peerUUID, "cluster-uuid=their-cluster", "ip=192.168.1.9"}, }}) - n, ok := d.dir.get(peerUUID) + n, ok := d.dir.get(nodeDiscoveryRecordID(RawNode{ID: "peer", TXT: []string{"uuid=" + peerUUID, "ip=192.168.1.9"}})) require.True(t, ok, "peer advertising a cluster principal was not recorded") - require.True(t, n.Clustered(), "peer not annotated as clustered (%v)", n) + require.True(t, n.Clustered()) // It leaves, so it re-advertises without a cluster principal: invitable again. d.onBrowse(DiscoveryEvent{Type: "updated", Node: RawNode{ ID: "peer", TXT: []string{"v=1", "uuid=" + peerUUID, "ip=192.168.1.9"}, }}) - n, ok = d.dir.get(peerUUID) + n, ok = d.dir.get(nodeDiscoveryRecordID(RawNode{ID: "peer", TXT: []string{"uuid=" + peerUUID, "ip=192.168.1.9"}})) require.True(t, ok, "peer dropped from the directory when it left its cluster") - assert.False(t, n.Clustered(), "peer still annotated as clustered (clusterUuid") -} - -// toDirectoryNodeTrusted annotates an entry the way onBrowse does at the moment -// it is folded in, so a test can reproduce "discovered at time T" faithfully. -func toDirectoryNodeTrusted(n noderec.DirectoryNode, d *daemon) noderec.DirectoryNode { - n.Trusted = d.mesh.HasPin(n.ClusterUUID) - return n -} - -func mustClusterUUID(t *testing.T, d *daemon, hostUUID string) string { - t.Helper() - n, ok := d.dir.get(hostUUID) - require.True(t, ok, "node (%v)", hostUUID) - return n.ClusterUUID + assert.False(t, n.Clustered()) } diff --git a/services/nvpair-node-scanner/connreuse_test.go b/services/nvpair-node-scanner/connreuse_test.go index bd96b00a7..ed58e86af 100644 --- a/services/nvpair-node-scanner/connreuse_test.go +++ b/services/nvpair-node-scanner/connreuse_test.go @@ -73,10 +73,9 @@ func TestModelsClient_ReusesPeerConnections(t *testing.T) { // pinned mTLS path instead of the plaintext self-fetch. const rounds = 5 for i := 0; i < rounds; i++ { - models, _, _, ok := d.fetchModels("localhost", port, "uuid-peer") - require.True(t, ok, "round (%v, %v, %v)", i, ok, models) - require.Len(t, models, 1, "round (%v, %v, %v)", i, ok, models) - require.Equal(t, "m", models[0], "round (%v, %v, %v)", i, ok, models) + inventory, ok := d.fetchModels("localhost", port, "uuid-peer") + require.True(t, ok, "round %d", i) + assert.Equal(t, []string{"m"}, inventory.models, "round %d", i) } mu.Lock() diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 7db6a843c..fb7e4d5aa 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -4,8 +4,9 @@ package main import ( + "bytes" "context" - "crypto/tls" + "crypto/x509" "encoding/json" "io" "log/slog" @@ -14,12 +15,14 @@ import ( "net/http" "os" "path/filepath" + "slices" "strconv" "strings" "sync" "time" "nvpair-shared/clustertrust" + "nvpair-shared/discovery" "nvpair-shared/mdns" "nvpair-shared/netmon" "nvpair-shared/netpick" @@ -49,8 +52,7 @@ type daemon struct { // registry-driven self directory entry (publishSelf), which is anchored to // our own identity rather than to hearing our own multicast. instance string - http *http.Client - // modelsHTTP fetches a peer's /v1/models. It gets a longer timeout than http + // modelsHTTP fetches the local /v1/models. It gets a longer timeout than node-info // (node-info): engine-manager's models sweep is bounded by its own, larger // deadline, so a peer with several engines can legitimately take longer than // a node-info probe. A shorter client would give up mid-sweep and silently @@ -59,14 +61,21 @@ type daemon struct { // modelsClients is the long-lived per-peer mTLS pool for LAN /v1/models // fetches. A throwaway Transport every 15s paid a full handshake each time. modelsClients *clustertrust.PeerClientPool - // tlsHTTP, when non-nil (the operator passed --client-cert/--client-key/ - // --ca-bundle), fetches node-info over HTTPS on the ni port instead of plain - // HTTP. Dormant by default (nil): node-info is plain per the transport policy, - // so this is opt-in scaffolding for a future TLS node-info rollout, gated only - // on the flags — never inferred from a node's cluster-uuid or a TXT hint. - tlsHTTP *http.Client - // nodeInfoOrigins queues same-origin requests before http.Client starts its - // timeout, while nodeInfoTransport enforces the matching connection bound. + // modelsWake coalesces newly verified peers into the next model sweep. + modelsWake chan struct{} + // nodeInfoClients uses exact cluster pins, independently of public display HTTP. + nodeInfoClients *clustertrust.PeerClientPool + // peerMu serializes candidate generations, proof invalidation, and accepted + // results. No network request runs while it is held. + peerMu sync.Mutex + candidates map[string]*discoveryCandidate + peerCertificates map[string]*x509.Certificate + authWake chan struct{} + authRetries telemetryRetryGate + + // nodeInfoOrigins queues verified same-origin reads before http.Client starts + // its timeout. Candidate authentication skips busy origins and retries later. + // The peer client pool enforces the matching connection bound. nodeInfoOrigins nodeInfoOriginGate // baseDir is the config base nodeid.Resolve reads identity from. Derived @@ -91,23 +100,14 @@ type daemon struct { // localInfo is supplied only by the owned worker IPC path, under selfMu. localInfo *noderec.NodeInfo - // lastInfo / lastModels cache each node's last successful enrichment (keyed by - // hostUuid) — GPU/CPU/memory from node-info and the model list from - // engine-manager — so a transient fetch failure reuses the prior values - // instead of blanking the node card. Same resilience the legacy scanner path - // had via its own lastInfo cache. - infoMu sync.Mutex - lastInfo map[string]NodeInfoResponse + // infoMu guards peer read timestamps, model inventories, and outage state. + infoMu sync.Mutex + // lastInfoAt records accepted node-info reads for authenticated peer liveness. lastInfoAt map[string]time.Time - lastModels map[string][]string - // lastModelsByEngine caches the per-engine attribution (engine name -> - // models) alongside the flat lastModels union, so a transient /v1/models - // miss reuses both rather than blanking the per-engine card. Keyed by - // hostUuid like the other enrichment caches. + // These model caches preserve owned-local inventory when publication rebuilds + // the local record during an outage. Remote records retain models in the directory. + lastModels map[string][]string lastModelsByEngine map[string]map[string][]string - // lastLoadedByEngine caches the per-engine loaded (in-memory) set from the - // same /v1/models fetch, so a transient miss reuses it rather than blanking - // a remote node's loaded state. Keyed by hostUuid. lastLoadedByEngine map[string]map[string][]string // nodeInfoDown holds the nodes whose last node-info enrichment failed, so // the outage is reported when it starts and when it ends rather than once @@ -144,8 +144,7 @@ type daemon struct { observed map[string]bool } -// nodeInfoFetchTimeout bounds each node-info enrichment HTTP(S) GET, shared by -// the plain and (opt-in) TLS clients. +// nodeInfoFetchTimeout bounds authenticated node-info requests after origin queuing. const nodeInfoFetchTimeout = 3 * time.Second const nodeInfoResponseDrainLimit = 64 << 10 @@ -164,29 +163,14 @@ const nodeInfoResponseDrainLimit = 64 << 10 // already written off is not worth waiting longer for here. const nodeInfoDialTimeout = reach.DefaultTimeout -// nodeInfoTransport is the transport both node-info clients dial through: the -// plain one built here and, when the operator opts enrichment onto HTTPS, the -// TLS one built in tls.go. The TLS handshake gets the same budget as the -// connect, for the same reason. -func nodeInfoTransport(tlsConfig *tls.Config) *http.Transport { - return &http.Transport{ - DialContext: (&net.Dialer{Timeout: nodeInfoDialTimeout}).DialContext, - TLSClientConfig: tlsConfig, - TLSHandshakeTimeout: nodeInfoDialTimeout, - MaxConnsPerHost: 1, - MaxIdleConnsPerHost: 1, - IdleConnTimeout: 90 * time.Second, - } -} - // modelsFetchTimeout bounds a peer /v1/models GET. It exceeds engine-manager's // own models-sweep cap (modelsTimeout, 5s) so the fetch waits for a bounded // sweep to finish instead of timing out partway through it. const modelsFetchTimeout = 6 * time.Second // peerRefreshInterval is the cadence of the mDNS-independent convergence loop -// (refreshPeersLoop). The event-driven enrichment still handles fast initial -// discovery; this is the background sweep that catches changes that never move +// (refreshPeersLoop). Newly authenticated peers wake model refresh immediately; +// this is the background sweep that catches changes that never move // the mDNS record (a model pull/delete, a late engine start) and announcements // that did move it but were not received (a peer joining or leaving a cluster). // It comfortably exceeds modelsFetchTimeout so a worst-case sweep finishes well @@ -322,10 +306,8 @@ func telemetryRetryDelay(consecutiveFailures int) time.Duration { return delay } -// newDaemon builds the daemon. tlsHTTP is nil unless the operator opted node-info -// enrichment onto HTTPS via the --client-cert/--client-key/--ca-bundle flags; nil -// keeps the default plain-HTTP fetch. -func newDaemon(codec *Codec, mesh *clustertrust.Mesh, clusterDir string, tlsHTTP *http.Client) *daemon { +// newDaemon builds the registry publisher and pinned peer discovery daemon. +func newDaemon(codec *Codec, mesh *clustertrust.Mesh, clusterDir string) *daemon { // nodeid reads /cluster/identity.json; --cluster-dir is that cluster // subdir, so its parent is the base. Empty -> nodeid's shared default. baseDir := "" @@ -355,11 +337,13 @@ func newDaemon(codec *Codec, mesh *clustertrust.Mesh, clusterDir string, tlsHTTP baseDir: baseDir, clusterDir: clusterDir, dir: newDirectory(), - http: &http.Client{Timeout: nodeInfoFetchTimeout, Transport: nodeInfoTransport(nil)}, modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, modelsClients: clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout), - tlsHTTP: tlsHTTP, - lastInfo: make(map[string]NodeInfoResponse), + modelsWake: make(chan struct{}, 1), + nodeInfoClients: newNodeInfoClients(mesh), + candidates: make(map[string]*discoveryCandidate), + peerCertificates: make(map[string]*x509.Certificate), + authWake: make(chan struct{}, 1), lastInfoAt: make(map[string]time.Time), lastModels: make(map[string][]string), lastModelsByEngine: make(map[string]map[string][]string), @@ -367,18 +351,11 @@ func newDaemon(codec *Codec, mesh *clustertrust.Mesh, clusterDir string, tlsHTTP nodeInfoDown: make(map[string]bool), lastActivityAt: make(map[string]time.Time), } - // Anti-flap: before evicting a node that's missed the mDNS threshold, - // confirm it still answers on node-info (or engine-manager as fallback). - // Inference ports (ol/lm) are never probed — they are the promoted proxies - // and SYN spam there contributed to accept-queue exhaustion on macOS. - // - // Key by uuid= (not the mDNS instance name): two hosts sharing a hostname but - // holding distinct UUIDs must not collapse into one directory entry (matching - // cluster-manager's browser). Self re-stamps are handled separately in - // reloadIdentity, which moves this node's own directory entry to the new uuid. + // Track connection candidates independently of identity claims. Promoted + // peers survive multicast gaps using authenticated reads or inference. d.browser = NewDiscovery(noderec.ServiceType, noderec.Domain, WithLivenessProbe(func(n RawNode) bool { return d.reachable(n) }), - WithKeyFunc(func(n RawNode) string { return UUIDFromTXT(n.TXT) }), + WithKeyFunc(nodeDiscoveryRecordID), ) if resp, err := mdns.NewResponder(instance, noderec.ServiceType, noderec.Domain, noderec.SRVPort, reg.txt()); err != nil { slog.Warn("mDNS advertising disabled for _nvpair-node", "err", err) @@ -411,12 +388,10 @@ func (d *daemon) run(ctx context.Context) { events := make(chan DiscoveryEvent, 32) go d.browser.Run(ctx, events) - // Converge peer model inventories independently of mDNS change events: the - // browse-driven enrichment above only re-runs when the raw mDNS record - // changes, but models live off the record (engine-manager's em /v1/models), - // so a pull/delete, a late engine start, or a missed announcement would - // otherwise leave a directory entry permanently stale. Stops with ctx. + // Refresh peer inventory periodically and models after authentication, so + // model changes converge independently of mDNS. Stops with ctx. go d.refreshPeersLoop(ctx) + go d.authenticateCandidatesLoop(ctx) go d.refreshTelemetryLoop(ctx) for { select { @@ -424,6 +399,9 @@ func (d *daemon) run(ctx context.Context) { if d.modelsClients != nil { d.modelsClients.CloseIdle() } + if d.nodeInfoClients != nil { + d.nodeInfoClients.CloseIdle() + } return case ev, ok := <-events: if !ok { @@ -434,158 +412,362 @@ func (d *daemon) run(ctx context.Context) { } } -// onBrowse folds one _nvpair-node browse event into the directory (annotating -// trust from the cluster pin set) and emits the matching discovery:node-* event. -func (d *daemon) onBrowse(ev DiscoveryEvent) { - // Never let a browse event touch our own entry. Self is registry-driven - // (publishSelf); folding self browse events in here is exactly what evicted - // the local node on Windows when its own multicast stopped looping back. - // Our record always carries uuid=, so this reliably matches. - if uuid := UUIDFromTXT(ev.Node.TXT); uuid != "" && uuid == d.reg.record().HostUUID { - return +type discoveryCandidate struct { + node noderec.DirectoryNode + // Retain the last authenticated source link for expiry while a changed + // announcement awaits proof. It does not authenticate the pending claims. + peerUUID string + pending bool +} + +type peerInfoSample struct { + info NodeInfoResponse + certificate *x509.Certificate + localCertificate []byte + receivedAt time.Time +} + +func newNodeInfoClients(mesh *clustertrust.Mesh) *clustertrust.PeerClientPool { + return clustertrust.NewPeerClientPoolOpts(mesh, clustertrust.PeerClientOptions{ + Timeout: nodeInfoFetchTimeout, DialTimeout: nodeInfoDialTimeout, MaxConnsPerHost: 1, + }) +} + +func (d *daemon) wakeAuthentication() { + select { + case d.authWake <- struct{}{}: + default: } +} - trusted := false - if cu := clustertrust.ClusterUUIDFromTXT(ev.Node.TXT); cu != "" { +// authenticateCandidatesLoop has its own permits and retry state. Pending +// discoveries cannot occupy the verified telemetry workers or their host memory. +func (d *daemon) authenticateCandidatesLoop(ctx context.Context) { + ticker := time.NewTicker(telemetryRefreshInterval) + defer ticker.Stop() + sem := make(chan struct{}, telemetryRefreshConcurrency) + var workers sync.WaitGroup + defer workers.Wait() + for { + select { + case <-ctx.Done(): + return + case <-d.authWake: + case <-ticker.C: + } d.mesh.Refresh() - trusted = d.mesh.HasPin(cu) + d.reconcileTrust() + d.dispatchCandidates(ctx, sem, &workers) } - node, ok := toDirectoryNode(ev.Node, trusted) - if !ok { - // Record without a uuid= — toDirectoryNode already warned. Nothing to - // store, and a removed event for it is a no-op since it was never added. - return +} + +func (d *daemon) dispatchCandidates(ctx context.Context, sem chan struct{}, workers *sync.WaitGroup) { + d.peerMu.Lock() + defer d.peerMu.Unlock() + for id, candidate := range d.candidates { + if !candidate.pending { + continue + } + select { + case sem <- struct{}{}: + default: + return + } + target := telemetryTargetKey(candidate.node.CandidateIPs(), candidate.node.Services[noderec.ServiceNodeInfo].Port) + candidate.node.ClusterUUID + token, due := d.authRetries.claim(id, target, time.Now()) + if !due { + <-sem + continue + } + workers.Add(1) + go func() { + defer workers.Done() + defer func() { <-sem }() + ok := d.authenticateCandidate(ctx, id, candidate) + d.authRetries.finish(id, token, ok, time.Now()) + }() } +} - var method string - switch ev.Type { - case "removed": - d.dir.remove(node.HostUUID) - d.forget(node.HostUUID) - method = noderec.NotifyNodeRemoved - case "updated": - d.enrich(&node) - d.supersedingUpsert(node) - method = noderec.NotifyNodeUpdated - default: // discovered - d.enrich(&node) - d.supersedingUpsert(node) - method = noderec.NotifyNodeDiscovered +func (d *daemon) authenticateCandidate(ctx context.Context, id string, candidate *discoveryCandidate) bool { + sample, ok := d.readPeerInfo(ctx, candidate.node) + if !ok { + return false } - d.emit(method, node) + d.peerMu.Lock() + defer d.peerMu.Unlock() + if d.candidates[id] != candidate || !d.currentPin(sample) { + return false + } + if sample.info.HostUUID == d.reg.record().HostUUID { + return false + } + candidate.peerUUID = sample.info.HostUUID + candidate.pending = false + d.publishPeerInfo(id, sample) + return true } -// supersedingUpsert folds a browsed peer into the directory, evicting any record -// it is PROVEN to have replaced and emitting the removals BEFORE the caller emits -// the node's own event — so no consumer ever observes the machine under two -// identities at once. -// -// The directory can only nominate (directory.supersedeCandidates). Its signals -// are the arriving record's instance name — an unauthenticated mDNS field -// anything on the LAN can claim — plus a LastSeen gap that a healthy peer's -// frozen timestamp satisfies exactly as readily as a ghost's. Deleting on that -// alone would make mDNS a removal primitive: a record claiming a live peer's -// hostname would evict it, the browser re-reports a node only when its record -// CHANGES so it might never come back, and the loss would carry through to the -// broker's onNodeLost and to workload routing. -// -// So each candidate is confirmed the way the eviction path confirms a -// threshold-missed node: ask the machines involved who they are, and delete only -// on a definite mismatch. A candidate that cannot be confirmed stays, and is not -// left uncovered — when it stops being advertised the ordinary miss-threshold -// path (reachable) puts the same question to it. -func (d *daemon) supersedingUpsert(node noderec.DirectoryNode) { - var proven []noderec.DirectoryNode - for _, old := range d.dir.supersedeCandidates(node, d.reg.record().HostUUID) { - if d.identityReplaced(old, node) { - proven = append(proven, old) - } +// readPeerInfo authenticates the exact endpoint and response together. Claimed +// host identity cannot switch a known host to another pinned principal. +func (d *daemon) readPeerInfo(ctx context.Context, node noderec.DirectoryNode) (peerInfoSample, bool) { + principal := node.ClusterUUID + if known, ok := d.dir.get(node.HostUUID); ok && known.IdentityVerification == noderec.IdentityPinned { + principal = known.ClusterUUID } - for _, old := range d.dir.upsertEvicting(node, proven) { - slog.Warn("dropping node record proven superseded by a newer identity on the same host", - "ip", old.IP, "droppedUuid", old.HostUUID, "droppedName", old.Name, - "keptUuid", node.HostUUID, "keptName", node.Name, - "ageSeconds", node.LastSeen-old.LastSeen) - d.forget(old.HostUUID) - d.emit(noderec.NotifyNodeRemoved, old) + if principal == "" { + principal = node.HostUUID + } + d.mesh.Refresh() + if principal == "" || principal != node.HostUUID || !d.mesh.HasPin(principal) || d.nodeInfoClients == nil { + return peerInfoSample{}, false + } + port := node.Services[noderec.ServiceNodeInfo].Port + if port < 1 || port > 65535 { + return peerInfoSample{}, false + } + config, ok := d.mesh.ClientTLSConfig(principal) + if !ok { + return peerInfoSample{}, false + } + ask := func(host string) (peerInfoSample, bool) { + started := time.Now() + origin := "https://" + net.JoinHostPort(host, strconv.Itoa(port)) + var release func() + var acquired bool + if node.IdentityVerification == noderec.IdentityPinned { + release, acquired = d.nodeInfoOrigins.acquire(ctx, origin) + } else { + release, acquired = d.nodeInfoOrigins.tryAcquire(origin) + } + if !acquired { + return peerInfoSample{}, false + } + defer release() + requestCtx, cancel := context.WithTimeout(ctx, nodeInfoFetchTimeout) + defer cancel() + client, ok := d.nodeInfoClients.Client(principal) + if !ok { + return peerInfoSample{}, false + } + req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, origin+"/v1/node-info", nil) + if err != nil { + return peerInfoSample{}, false + } + resp, err := client.Do(req) + if err != nil { + return peerInfoSample{}, false + } + defer func() { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, nodeInfoResponseDrainLimit)) + _ = resp.Body.Close() + }() + if resp.StatusCode != http.StatusOK || resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { + return peerInfoSample{}, false + } + var info NodeInfoResponse + body, err := io.ReadAll(io.LimitReader(resp.Body, nodeInfoResponseDrainLimit+1)) + if err != nil || len(body) > nodeInfoResponseDrainLimit { + return peerInfoSample{}, false + } + if err := json.Unmarshal(body, &info); err != nil { + return peerInfoSample{}, false + } + if info.HostUUID != principal || info.ClusterUUID == nil || *info.ClusterUUID != principal || !validPeerMetadata(info.Peer) { + return peerInfoSample{}, false + } + sample := peerInfoSample{info: info.Aged(time.Since(started)), certificate: resp.TLS.PeerCertificates[0], localCertificate: config.Certificates[0].Certificate[0], receivedAt: time.Now()} + if !d.currentPin(sample) { + return peerInfoSample{}, false + } + return sample, true } + // Candidate address memory must never replace a verified host's target. + key := node.DiscoveryRecordID + _, sample, ok := askRemembered(&d.enrichHosts, hostKey{hostUUID: key, service: noderec.ServiceNodeInfo}, node.CandidateIPs(), ask) + return sample, ok } -// identityReplaced reports whether old is PROVEN to be a pre-wipe ghost of the -// machine now arriving as replacement — the confirmation supersedingUpsert -// requires before it deletes anything. -// -// It takes two answers, both from node-info, and both are required: -// -// - old must not still be answering for itself. Every address old published is -// asked, at old's own ni port, and any of them naming old keeps it. This is -// what tells a wipe apart from two hosts that merely share a hostname, and it -// is why a record with no ni port to ask is never evicted here: a candidate -// that cannot answer cannot be distinguished from a live namesake, and the -// miss-threshold path will reach it on its own when it stops advertising. -// - the machine at replacement's address must name someone other than old. -// That is the arriving record's own port, which is where the machine is -// answering now. -// -// Anything short of both — no ni port, no answer within the probe budget, a -// response without a hostUuid, or the 403 a clustered peer returns to a plaintext -// caller — leaves the record in place. -func (d *daemon) identityReplaced(old, replacement noderec.DirectoryNode) bool { - oldNI, ok := old.Services[noderec.ServiceNodeInfo] - if !ok || oldNI.Port == 0 { +func validPeerMetadata(peer *noderec.PeerMetadata) bool { + if peer == nil || peer.Name == "" || len(peer.Addresses) == 0 || len(peer.Addresses) > noderec.MaxAdvertisedIPs { return false } - stillItself, _ := d.identityAlive(noderec.NodeRecord{ - HostUUID: old.HostUUID, - IP: old.IP, - Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: oldNI.Port}, - }, old.CandidateIPs()) - if stillItself { + for _, ip := range peer.Addresses { + if net.ParseIP(ip) == nil { + return false + } + } + if peer.Services[noderec.ServiceNodeInfo] == 0 { return false } - ni, ok := replacement.Services[noderec.ServiceNodeInfo] - if !ok || ni.Port == 0 { + for _, port := range peer.Services { + if port < 1 || port > 65535 { + return false + } + } + return true +} + +func (d *daemon) currentPin(sample peerInfoSample) bool { + d.mesh.Refresh() + config, ok := d.mesh.ClientTLSConfig(sample.info.HostUUID) + return ok && bytes.Equal(sample.localCertificate, config.Certificates[0].Certificate[0]) && d.mesh.VerifyPeerCertificate(sample.info.HostUUID, sample.certificate) +} + +// currentPeerResult also rejects results for an endpoint replaced while a read +// was queued or in flight. peerMu must be held through the subsequent write. +func (d *daemon) currentPeerResult(target noderec.DirectoryNode, sample peerInfoSample) bool { + if !d.currentPin(sample) { return false } - // Ask the arriving machine at every address it ranked, not just its canonical - // one. Any of them naming old keeps the record, so a wider question can only - // spare a live node; asking one address that this host happens not to reach - // yields no answer at all, and a record with only ips= has no canonical - // address to ask. - replacementIPs := replacement.CandidateIPs() - rec := noderec.NodeRecord{ - HostUUID: old.HostUUID, - IP: replacement.IP, - Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: ni.Port}, - } - alive, conclusive := d.identityAlive(rec, replacementIPs) - return conclusive && !alive + node, ok := d.dir.get(target.HostUUID) + return ok && node.IdentityVerification == noderec.IdentityPinned && node.DiscoveryRecordID == target.DiscoveryRecordID && node.ClusterUUID == target.ClusterUUID && node.Services[noderec.ServiceNodeInfo].Port == target.Services[noderec.ServiceNodeInfo].Port && slices.Equal(node.CandidateIPs(), target.CandidateIPs()) && d.mesh.VerifyPeerCertificate(node.ClusterUUID, d.peerCertificates[node.HostUUID]) } -// reloadTrust re-derives everything this daemon caches from the cluster pin set, -// in response to nvpair-cluster-manager reporting that the set moved -// (discovery:reload-trust). -// -// It exists because the browse path is not a sufficient trigger. A peer's trusted -// annotation is computed when its mDNS record is folded in, and the Browser only -// reports a node whose record actually changed — so a peer that was already -// advertising its cluster principal when this node joined is annotated from -// before our own pins existed and never revisited. That left it unroutable for -// the life of the process while the roster and the pins on disk both said -// otherwise. -// -// The announcement comes from the process that writes the pins, and only after -// the write lands, so the directory read here is never older than the event that -// prompted it. Recomputing unconditionally is deliberate: gating on any -// "did it change?" helper is how the previous membership hook silently did -// nothing. +func (d *daemon) recordPeerInfoSuccess(uuid string) { + d.infoMu.Lock() + defer d.infoMu.Unlock() + if d.lastInfoAt == nil { + d.lastInfoAt = make(map[string]time.Time) + } + d.lastInfoAt[uuid] = time.Now() +} + +// publishPeerInfo stores authenticated inventory and emits directory changes. +// The caller must hold peerMu to serialize proof, cache, and directory updates +// with peer invalidation. +func (d *daemon) publishPeerInfo(candidateID string, sample peerInfoSample) { + info := sample.info.Aged(time.Since(sample.receivedAt)) + services := make(map[noderec.ServiceKey]noderec.ServiceStatus, len(info.Peer.Services)) + for key, port := range info.Peer.Services { + services[key] = noderec.ServiceStatus{Port: port} + } + node := noderec.DirectoryNode{ + DiscoveryRecordID: "peer:" + info.HostUUID, IdentityVerification: noderec.IdentityPinned, + HostUUID: info.HostUUID, ClusterUUID: *info.ClusterUUID, Trusted: true, + Name: info.Peer.Name, IP: info.Peer.Addresses[0], IPs: info.Peer.Addresses, Services: services, + GPUs: info.GPUs, CPU: info.CPU, Memory: info.Memory, + } + if d.peerCertificates == nil { + d.peerCertificates = make(map[string]*x509.Certificate) + } + d.peerCertificates[info.HostUUID] = sample.certificate + d.recordPeerInfoSuccess(info.HostUUID) + removed, discovered, changed := d.dir.promote(candidateID, node) + if removed.DiscoveryRecordID != "" { + d.emit(noderec.NotifyNodeRemoved, removed) + } + if changed { + method := noderec.NotifyNodeUpdated + if discovered { + method = noderec.NotifyNodeDiscovered + } + current, _ := d.dir.get(info.HostUUID) + d.emit(method, current) + } + if discovered { + select { + case d.modelsWake <- struct{}{}: + default: + } + } + var utilization uint32 + for _, gpu := range info.GPUs { + if gpu.UtilizationPercent > utilization { + utilization = gpu.UtilizationPercent + } + } + info = sample.info.Aged(time.Since(sample.receivedAt)) + d.emitTelemetry(noderec.NodeTelemetry{HostUUID: info.HostUUID, GPUUtilizationPct: utilization, TelemetryValid: info.TelemetryValid, MSSince: info.MSSince}) +} + +// onBrowse stores each discovery independently. Network claims never mutate a +// canonical host, including when a discovery changes or disappears. +func (d *daemon) onBrowse(ev DiscoveryEvent) { + var node noderec.DirectoryNode + alive := false + if ev.Type == discovery.Removed { + // Find the stored discovery even if its latest announcement changed or + // dropped the UUID claim. Cleanup must not depend on validating that claim. + d.peerMu.Lock() + candidate := d.candidates[nodeDiscoveryRecordID(ev.Node)] + peerUUID := "" + if candidate != nil { + node = candidate.node + peerUUID = candidate.peerUUID + } + d.peerMu.Unlock() + if candidate == nil { + return + } + // Judge the canonical peer separately from the expiring observation. + // A failed update may still have an older authenticated source link. + alive = d.peerReachable(peerUUID) + } else { + if UUIDFromTXT(ev.Node.TXT) == d.reg.record().HostUUID { + return + } + var ok bool + node, ok = toDirectoryNode(ev.Node) + if !ok { + return + } + } + d.peerMu.Lock() + defer d.peerMu.Unlock() + if ev.Type == discovery.Removed { + candidate := d.candidates[node.DiscoveryRecordID] + delete(d.candidates, node.DiscoveryRecordID) + d.authRetries.forget(node.DiscoveryRecordID) + if d.dir.remove(node.DiscoveryRecordID) { + d.emit(noderec.NotifyNodeRemoved, node) + } + if candidate != nil && candidate.peerUUID != "" && !alive { + // Another authenticated discovery still represents this peer. + for _, other := range d.candidates { + if other.peerUUID == candidate.peerUUID { + return + } + } + if peer, present := d.dir.get(candidate.peerUUID); present { + d.dir.remove(peer.HostUUID) + delete(d.peerCertificates, peer.HostUUID) + d.forget(peer.HostUUID) + d.emit(noderec.NotifyNodeRemoved, peer) + } + } + return + } + if d.candidates == nil { + d.candidates = make(map[string]*discoveryCandidate) + } + peerUUID := "" + if previous := d.candidates[node.DiscoveryRecordID]; previous != nil { + peerUUID = previous.peerUUID + } + d.candidates[node.DiscoveryRecordID] = &discoveryCandidate{node: node, peerUUID: peerUUID, pending: true} + method := noderec.NotifyNodeUpdated + if d.dir.upsert(node) { + method = noderec.NotifyNodeDiscovered + } + d.emit(method, node) + d.wakeAuthentication() +} + +// reloadTrust invalidates stale proofs and schedules authentication after a pin change. func (d *daemon) reloadTrust() { d.mesh.Refresh() // A pin can flip membership on its own (a cluster dir predating admission.json // is clustered on pins alone), which changes what we advertise, so settle - // identity before annotating peers against it. + // identity before reconciling peer proofs. d.reconcileIdentity() d.reconcileTrust() + d.peerMu.Lock() + for id := range d.candidates { + d.authRetries.forget(id) + } + d.peerMu.Unlock() + d.wakeAuthentication() } // reconcileIdentity re-advertises this node when live membership no longer @@ -609,31 +791,33 @@ func (d *daemon) reconcileIdentity() { d.reloadIdentity() } -// reconcileTrust re-evaluates every directory entry's trusted annotation against -// the pin set the mesh currently holds, emitting one discovery:node-updated per -// entry whose answer changed. Entries that agree are left untouched and silent, -// so the steady state is a map lookup per node and no traffic — a peer paired or -// removed since the last pass is the only thing that produces an event. -// -// Self is included: it sits in the directory like any other entry, and -// Mesh.HasPin answers for our own principal through the same self-trust path -// publishSelf uses, so both stay consistent. -// -// The caller refreshes the mesh; this reads whatever it currently holds. It is -// idempotent, so a duplicated or coalesced announcement costs one map lookup per -// known node and emits nothing. +// reconcileTrust removes proofs whose exact pin is no longer current. +// A new pin schedules authentication; it cannot make an announcement trusted. func (d *daemon) reconcileTrust() { - for _, n := range d.dir.snapshot("") { - // nil principal: this pass knows the pin set, not what each peer - // advertises, so it re-derives trust from whatever principal is stored - // when the write lands rather than from the snapshot it is iterating. - node, changed := d.dir.applyClusterIdentity(n.HostUUID, nil, d.mesh.HasPin) - if !changed { + d.peerMu.Lock() + defer d.peerMu.Unlock() + for uuid, cert := range d.peerCertificates { + if d.mesh.VerifyPeerCertificate(uuid, cert) { continue } - slog.Info("node trust changed", "host_uuid", n.HostUUID, - "cluster_uuid", n.ClusterUUID, "trusted", node.Trusted) - d.emit(noderec.NotifyNodeUpdated, node) + delete(d.peerCertificates, uuid) + for _, candidate := range d.candidates { + if candidate.peerUUID == uuid { + candidate.peerUUID = "" + candidate.pending = true + } + } + if node, ok := d.dir.get(uuid); ok { + d.dir.remove(uuid) + d.forget(uuid) + d.emit(noderec.NotifyNodeRemoved, node) + } + } + if d.nodeInfoClients != nil { + d.nodeInfoClients.DropUnpinned() + } + if d.modelsClients != nil { + d.modelsClients.DropUnpinned() } } @@ -767,7 +951,7 @@ func (d *daemon) peerAddresses() []string { } var out []string for _, n := range d.dir.snapshot("") { - if n.HostUUID == self { + if n.HostUUID == self || n.IdentityVerification != noderec.IdentityPinned { continue } if n.IP != "" { @@ -802,17 +986,14 @@ func (d *daemon) refreshAdvertisedAddressesOnce() { } // dropSelf removes this node's stale directory entry after a uuid re-stamp, -// carrying its enrichment cache onto the new uuid so the republish below reuses -// it (and can't blank the card), and emits removed(old) so consumers reconcile. +// carries its model caches onto the new uuid, and emits removed(old) so consumers +// reconcile. // The new entry is (re)published by the caller via publishSelfLocked. Must be // called with selfMu held. func (d *daemon) dropSelf(oldHostUUID, newHostUUID string) { node, existed := d.dir.get(oldHostUUID) d.dir.remove(oldHostUUID) d.infoMu.Lock() - if info, ok := d.lastInfo[oldHostUUID]; ok { - d.lastInfo[newHostUUID] = info - } if at, ok := d.lastInfoAt[oldHostUUID]; ok { d.lastInfoAt[newHostUUID] = at } @@ -825,7 +1006,6 @@ func (d *daemon) dropSelf(oldHostUUID, newHostUUID string) { if loaded, ok := d.lastLoadedByEngine[oldHostUUID]; ok { d.lastLoadedByEngine[newHostUUID] = loaded } - delete(d.lastInfo, oldHostUUID) delete(d.lastInfoAt, oldHostUUID) delete(d.lastModels, oldHostUUID) delete(d.lastModelsByEngine, oldHostUUID) @@ -888,14 +1068,28 @@ func (d *daemon) publishSelfLocked() { if d.localInfo != nil && d.localInfo.HostUUID == rec.HostUUID { node.GPUs, node.CPU, node.Memory = d.localInfo.GPUs, d.localInfo.CPU, d.localInfo.Memory } - // Models still come from the locally owned engine-manager over loopback. - d.enrichModelsCandidates(&node, []string{loopbackHost}) + // Publication never waits for HTTP while holding selfMu. The model sweep + // refreshes the owned engine-manager independently and retains its last answer. + // Serialize the cache copy and publication with its model apply/cache write. + d.peerMu.Lock() + defer d.peerMu.Unlock() + if node.HasService(noderec.ServiceEngineManager) { + d.infoMu.Lock() + node.Models = d.lastModels[rec.HostUUID] + node.ModelsByEngine = d.lastModelsByEngine[rec.HostUUID] + node.LoadedByEngine = d.lastLoadedByEngine[rec.HostUUID] + d.infoMu.Unlock() + } method := noderec.NotifyNodeUpdated if isNew := d.dir.upsert(node); isNew { method = noderec.NotifyNodeDiscovered } d.emit(method, node) + select { + case d.modelsWake <- struct{}{}: + default: + } } // acceptLocalNodeInfo applies an owned IPC snapshot to its matching local @@ -950,23 +1144,14 @@ func (d *daemon) watchAddress(ctx context.Context) { } } -// reachableTimeout bounds each liveness probe attempt (node-info GET or em TCP). -const reachableTimeout = time.Second - -// probeIdentityTimeout bounds the liveness probe's node-info read. It is the -// probe's own budget, deliberately not the enrichment client's -// nodeInfoFetchTimeout: enrichment is decorating a card and can afford to wait, -// whereas this runs per threshold-missed node inside the browser's scan cycle -// (shared/discovery). It matches reachableTimeout because the peer -// has already answered a TCP connect by the time we ask — a node that is up and -// listening but cannot serve node-info within a second is one we treat as -// unidentifiable and fall back on, not one worth stalling the scan for. -const probeIdentityTimeout = time.Second - // reachableInfoFreshness is how long a successful node-info enrichment keeps a -// threshold-missed node without a new probe (~2× default discovery scan interval). +// threshold-missed discovery (~2× default discovery scan interval). const reachableInfoFreshness = 10 * time.Second +// retainedPeerGrace gives peers whose last discovery expired a minute without +// authenticated inventory before removal. Recent inference also keeps them. +const retainedPeerGrace = time.Minute + // activityFreshness is how long a peer's reported inference traffic vouches for // it without a probe of its own. // @@ -986,278 +1171,48 @@ const activityFreshness = 60 * time.Second // node-info inventory and telemetry arrive over private worker IPC. const loopbackHost = "127.0.0.1" -// reachable reports whether a threshold-missed node is still THERE — an mDNS -// miss is not proof a node is gone, so a missed record is probed before it is -// evicted. -// -// Two kinds of evidence can vouch for a node without dialing it: a fresh -// lastInfo enrichment, and fresh inference traffic reported by the local proxies -// (lastActivityAt). Otherwise it answers in two stages, cheap then precise. -// -// First the TCP sweep: dial node-info (ni) then engine-manager (em) and stop at -// the first that connects. Inference ports (ol/lm) are never dialed — they are -// the promoted proxies and SYN spam there contributed to accept-queue -// exhaustion. Nothing answering means the node is gone, and no further question -// is worth asking — that is the ordinary departure, and it is settled without -// any HTTP. -// -// Something answering is NOT the end of it, and this is the bug that made a -// wiped node permanent. A connect only proves SOMETHING is listening at an -// address, not that it is the node the record describes. A node whose appdata is -// wiped mints a fresh hostUuid but keeps its LAN address and re-binds the same -// fixed service ports, so the sweep reaches the machine's NEW incarnation and -// reports the OLD record alive. The miss counter is reset on every pass and the -// stale record never ages out — the fleet carries a permanent duplicate of one -// machine. So when something does answer, we ask the machine who it is -// (identityAlive) and let a definite answer overrule the connect. -// -// Running the sweep first is what keeps that identity check off the hot path: it -// only runs when the node would otherwise have been KEPT, which is exactly when -// the two cases need telling apart. Ordering the other way would spend a -// node-info timeout on every departed node inside the browser's scan cycle. The -// verdicts are identical either way — a conclusive identity answer -// requires node-info to have responded, so its port is open, so the sweep would -// have succeeded regardless. -// -// A node advertising neither ni nor em (identity-only, or ol/lm alone) has -// nothing to probe and evicts normally. +// reachable retains a promoted discovery using its canonical peer's authenticated +// inventory or approved inference activity. An open port is not identity proof. func (d *daemon) reachable(n RawNode) bool { - rec := noderec.ParseTXT(n.TXT) - if rec.HostUUID != "" && d.reg != nil && rec.HostUUID == d.reg.record().HostUUID { - return false // an mDNS advertisement cannot establish this host's local identity - } - candidates := netpick.Candidates(n.TXT, n.Addresses) - if len(candidates) == 0 { - return false - } - - if rec.HostUUID != "" { - d.infoMu.Lock() - at, had := d.lastInfoAt[rec.HostUUID] - d.infoMu.Unlock() - if had && time.Since(at) < reachableInfoFreshness { - return true - } - if since, ok := d.activitySince(rec.HostUUID); ok && since < activityFreshness { - slog.Debug("liveness probe skipped: the node was recently serving inference", - "host_uuid", rec.HostUUID, "last_activity_ago", since) - return true - } - } - - answered := false - answeredIP := "" - answeredPort := 0 - for _, ip := range candidates { - for _, svc := range []noderec.ServiceKey{noderec.ServiceNodeInfo, noderec.ServiceEngineManager} { - port, ok := rec.Port(svc) - if !ok { - continue - } - conn, err := net.DialTimeout("tcp", net.JoinHostPort(ip, strconv.Itoa(port)), reachableTimeout) - if err == nil { - _ = conn.Close() - answered = true - answeredIP = ip - answeredPort = port - break - } - } - if answered { - break - } - } - if !answered { + // An unverified observation cannot borrow an established host's liveness. + d.peerMu.Lock() + candidate := d.candidates[nodeDiscoveryRecordID(n)] + if candidate == nil || candidate.pending || candidate.peerUUID == "" { + d.peerMu.Unlock() return false } - if alive, conclusive := d.identityAlive(rec, candidates); conclusive { - return alive - } - // Kept on the sweep's word alone. Record which port answered next to the - // node-info port we then failed to read: the pair separates "node-info is - // down but engine-manager answered" from "node-info accepted the connection - // and did not respond in time", which are indistinguishable in a report that - // only says the node flickered. - if niPort, ok := rec.Port(noderec.ServiceNodeInfo); ok { - slog.Debug("liveness probe kept a node without settling its identity", - "dial_ip", answeredIP, "advertised_ip", rec.IP, "answered_port", answeredPort, "ni_port", niPort) - } - return true + uuid := candidate.peerUUID + d.peerMu.Unlock() + return d.peerReachable(uuid) } -// identityAlive asks the machines at ips whether any of them is still the node rec -// describes, by reading hostUuid from node-info. In the eviction path it is called -// only after the TCP sweep has already found an address answering, so its job is -// to overrule that verdict when the machine turns out to be someone else. -// -// It walks every address rather than only the record's first because one wrong -// answer is not proof of a wipe: a direct-connect link uses the same subnet on -// every pair of machines that has one, so a record's leading address can resolve -// locally to a different host entirely. An address naming the record's own host -// settles it in the record's favour, and a mismatch counts only once no address -// has. -// -// conclusive reports whether the question could be answered at all. It is false -// — meaning "keep the TCP sweep's verdict" — when the record advertises no ni -// port, when it carries no uuid= to compare against, when no address answers -// within probeIdentityTimeout, or when every answer omits hostUuid. Only a -// node-info response that actually names a host is treated as proof, in either -// direction. -// -// Two properties bound what this can get wrong, and both are worth knowing -// before changing it: -// -// - It only ever concludes for an UNCLUSTERED peer. A clustered node's -// node-info admits pinned mTLS callers only (nvpair-node-info's handler -// refuses everyone else with 403), so this plaintext probe simply fails -// against a member and falls back to the TCP sweep. That is the right -// coverage for this bug: a wiped node is unclustered by definition, so the -// case we need to catch is exactly the case we can see. -// - It assumes node-info's hostUuid and the scanner's advertised uuid= name -// the same identity. They are deliberately kept in agreement — the broker -// passes its resolved node id to node-info, and cluster-manager ADOPTS an -// existing node-id rather than minting a rival one (see its -// TestFirstMintAdoptsExistingNodeUUID). If that invariant is ever broken, -// this probe would read the skew as a wipe and evict a live peer. -func (d *daemon) identityAlive(rec noderec.NodeRecord, ips []string) (alive, conclusive bool) { - if rec.HostUUID == "" || (d.http == nil && d.tlsHTTP == nil) { - return false, false - } - port, ok := rec.Port(noderec.ServiceNodeInfo) - if !ok { - return false, false - } - mismatchedAt, mismatchedUUID := "", "" - for _, ip := range ips { - if ip == "" { - continue - } - info, ok := d.probeIdentity(ip, port) - if !ok || info.HostUUID == "" { - continue - } - if info.HostUUID == rec.HostUUID { - return true, true - } - mismatchedAt, mismatchedUUID = ip, info.HostUUID - } - if mismatchedAt == "" { - return false, false - } - // Every address that answered named someone else, so the record describes a - // node that no longer exists: the machine was reset, or the address was - // reassigned. Warn — this is the signal that a node was wiped, and it is the - // only place the fleet can observe it. - slog.Warn("node record superseded: no address it published still answers for it", - "ip", mismatchedAt, "recordUuid", rec.HostUUID, "actualUuid", mismatchedUUID) - return false, true -} - -// probeIdentity reads one address's node-info under the identity probe's own -// budget, which is tighter than enrichment's: this runs inside the browser's scan -// cycle, and each address gets the same allowance so a slow first candidate cannot -// consume the rest of the list's time. -func (d *daemon) probeIdentity(ip string, port int) (NodeInfoResponse, bool) { - ctx, cancel := context.WithTimeout(context.Background(), probeIdentityTimeout) - defer cancel() - return d.fetchNodeInfoWithin(ctx, ip, port) -} - -// enrich decorates a node with the data that lives off the mDNS record: its -// node-info inventory (GPU/CPU/memory), fetched over plain HTTP per the transport -// policy, and its engine-manager model list, which is cluster data and so is -// fetched from a peer only over cluster mTLS (see fetchModels). Both are -// independent (a node may advertise one without the other). Peers are dialed at -// each address they published until one answers for the expected identity. -func (d *daemon) enrich(node *noderec.DirectoryNode) { - hosts := node.CandidateIPs() - d.enrichInfoCandidates(node, hosts) - d.enrichModelsCandidates(node, hosts) -} - -// enrichInfoAt decorates a node that advertises node-info (ni) with its -// GPU/CPU/memory inventory, fetched at the ni port on host. By default this is -// PLAIN HTTP — node-info is plain per the transport policy, and the daemon must -// NEVER infer mTLS from the node-level cluster-uuid= (the subtlest correctness -// requirement in the consolidation). The only way TLS is used is if the operator -// opts in globally via the --client-cert/--client-key/--ca-bundle flags (see -// fetchNodeInfo). A transient fetch failure reuses the node's last successful -// metrics rather than blanking its card; a node never enriched stays un-enriched. -func (d *daemon) enrichInfoAt(node *noderec.DirectoryNode, host string) { - d.enrichInfoCandidates(node, []string{host}) -} - -// enrichInfoCandidates is enrichInfoAt with address failover. A raw TCP accept -// is not enough here: identically configured direct-connect links can reuse an -// address on different machines, so the response's host UUID is checked before -// the candidate is accepted. Older peers that omit hostUuid remain compatible. -// -// Which address is asked, and in what order, is askRemembered's business: the one -// that answered last sweep by itself, and the rest together only if it stopped. -func (d *daemon) enrichInfoCandidates(node *noderec.DirectoryNode, hosts []string) { - ni, ok := node.Services[noderec.ServiceNodeInfo] - if !ok || len(hosts) == 0 { - return +func (d *daemon) peerReachable(uuid string) bool { + if uuid == "" { + return false } - ask := func(host string) (NodeInfoResponse, bool) { - candidate, fetched := d.fetchNodeInfo(host, ni.Port) - if !fetched { - return NodeInfoResponse{}, false - } - if candidate.HostUUID != "" && node.HostUUID != "" && candidate.HostUUID != node.HostUUID { - slog.Debug("node-info candidate answered for a different host; trying next", - "expected_host_uuid", node.HostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host) - return NodeInfoResponse{}, false - } - return candidate, true + d.peerMu.Lock() + node, ok := d.dir.get(uuid) + cert := d.peerCertificates[uuid] + d.mesh.Refresh() + valid := ok && d.mesh.VerifyPeerCertificate(node.ClusterUUID, cert) + d.peerMu.Unlock() + if !valid { + return false } - key := hostKey{hostUUID: node.HostUUID, service: noderec.ServiceNodeInfo} - selectedHost, info, _ := askRemembered(&d.enrichHosts, key, hosts, ask) - - endpointHost := selectedHost - if endpointHost == "" { - endpointHost = hosts[0] + d.infoMu.Lock() + at := d.lastInfoAt[node.HostUUID] + d.infoMu.Unlock() + if !at.IsZero() && time.Since(at) < reachableInfoFreshness { + return true } - d.noteNodeInfo(node.HostUUID, net.JoinHostPort(endpointHost, strconv.Itoa(ni.Port)), selectedHost != "") - if selectedHost != "" { - d.infoMu.Lock() - if d.lastInfo == nil { - d.lastInfo = make(map[string]NodeInfoResponse) - } - d.lastInfo[node.HostUUID] = info - if d.lastInfoAt == nil { - d.lastInfoAt = make(map[string]time.Time) - } - d.lastInfoAt[node.HostUUID] = time.Now() - d.infoMu.Unlock() - } else { - d.infoMu.Lock() - cached, had := d.lastInfo[node.HostUUID] - d.infoMu.Unlock() - if !had { - return - } - info = cached + if since, ok := d.activitySince(node.HostUUID); ok && since < activityFreshness { + return true } - node.GPUs = info.GPUs - node.CPU = info.CPU - node.Memory = info.Memory + return false } -// noteNodeInfo reports a peer's node-info endpoint going quiet, and coming back, -// at a level someone reading the log will see. -// -// Only the transitions are reported. The enrichment sweep repeats every -// peerRefreshInterval for as long as a node is advertised, so a peer whose -// address accepts nothing produces an unbounded run of identical failures; one -// line each time is why this was a debug line, and being a debug line is why an -// address that never once answered looked, in the log, exactly like one that was -// working. A node that is failing is stated once, with where it was asked, and -// its recovery is stated once. -// -// Silence between those two lines does not mean the peer is fine — it means -// nothing has changed and the node's card is still showing the last inventory -// that was actually read. +// noteNodeInfo logs outage and recovery transitions once each. Failed samples +// leave the last accepted inventory in place while scheduling measurements age. func (d *daemon) noteNodeInfo(hostUUID, endpoint string, ok bool) { d.infoMu.Lock() was := d.nodeInfoDown[hostUUID] @@ -1280,151 +1235,30 @@ func (d *daemon) noteNodeInfo(hostUUID, endpoint string, ok bool) { } } -// fetchNodeInfo does one GET of a node's /v1/node-info, returning the parsed -// inventory and whether it succeeded. It uses plain HTTP by default; when the -// operator configured a TLS client (--client-cert/--client-key/--ca-bundle) it -// fetches over HTTPS on the same ni port. The choice is purely flag-gated — never -// per-node — so the transport policy stays static. -func (d *daemon) fetchNodeInfo(ip string, port int) (NodeInfoResponse, bool) { - return d.fetchNodeInfoWithin(context.Background(), ip, port) -} - -// fetchNodeInfoWithin is fetchNodeInfo bounded by ctx as well as the client's own -// timeout. The liveness probe uses it to hold node-info to a tighter budget than -// enrichment without needing a second HTTP client (which would have to duplicate -// the TLS choice above); every other caller passes a background context and gets -// the client timeout unchanged. -func (d *daemon) fetchNodeInfoWithin(ctx context.Context, ip string, port int) (NodeInfoResponse, bool) { - client, scheme := d.http, "http" - if d.tlsHTTP != nil { - client, scheme = d.tlsHTTP, "https" - } - if client == nil { - return NodeInfoResponse{}, false - } - origin := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port)) - release, acquired := d.nodeInfoOrigins.acquire(ctx, origin) - if !acquired { - return NodeInfoResponse{}, false - } - defer release() - - url := origin + "/v1/node-info" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - slog.Debug("node-info request build failed", "ip", ip, "url", url, "err", err) - return NodeInfoResponse{}, false - } - resp, err := client.Do(req) - if err != nil { - slog.Debug("node-info fetch failed", "ip", ip, "url", url, "err", err) - return NodeInfoResponse{}, false - } - defer func() { - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, nodeInfoResponseDrainLimit)) - _ = resp.Body.Close() - }() - if resp.StatusCode != http.StatusOK { - return NodeInfoResponse{}, false - } - var info NodeInfoResponse - if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { - return NodeInfoResponse{}, false - } - return info, true -} - -// enrichModelsAt decorates a node that advertises engine-manager (em) with its -// model list, fetched at the em port (/v1/models) on host. The list moved off the -// size-limited mDNS TXT onto HTTP so it can grow unbounded and be owned by -// engine-manager (which already owns engine state). A peer is fetched over cluster -// mTLS pinned to the principal it advertises, and skipped when this node holds no -// pin for it — which models a machine holds is cluster data. A transient failure -// reuses the last successful list rather than blanking it; a node advertising no em -// has no model list (nothing is carried in the record). Self is dialed over -// loopback (see publishSelfLocked), which stays plaintext, peers at their advertised address. -func (d *daemon) enrichModelsAt(node *noderec.DirectoryNode, host string) { - d.enrichModelsCandidates(node, []string{host}) -} - -// enrichModelsCandidates is enrichModelsAt with address failover. For a peer, -// fetchModels performs the pinned mTLS handshake, so a shared address belonging -// to another machine is rejected and the next published address is tried. -func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []string) { - em, ok := node.Services[noderec.ServiceEngineManager] - if !ok || len(hosts) == 0 { - return - } - ask := func(host string) (modelInventory, bool) { - models, byEngine, loadedByEngine, fetched := d.fetchModels(host, em.Port, node.ClusterUUID) - if !fetched { - return modelInventory{}, false - } - return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true - } - key := hostKey{hostUUID: node.HostUUID, service: noderec.ServiceEngineManager} - if _, inventory, ok := askRemembered(&d.enrichHosts, key, hosts, ask); ok { - d.infoMu.Lock() - if d.lastModels == nil { - d.lastModels = make(map[string][]string) - } - if d.lastModelsByEngine == nil { - d.lastModelsByEngine = make(map[string]map[string][]string) - } - if d.lastLoadedByEngine == nil { - d.lastLoadedByEngine = make(map[string]map[string][]string) - } - d.lastModels[node.HostUUID] = inventory.models - d.lastModelsByEngine[node.HostUUID] = inventory.byEngine - d.lastLoadedByEngine[node.HostUUID] = inventory.loadedByEngine - d.infoMu.Unlock() - node.Models = inventory.models - node.ModelsByEngine = inventory.byEngine - node.LoadedByEngine = inventory.loadedByEngine - return - } - d.infoMu.Lock() - cached, had := d.lastModels[node.HostUUID] - cachedByEngine := d.lastModelsByEngine[node.HostUUID] - cachedLoaded := d.lastLoadedByEngine[node.HostUUID] - d.infoMu.Unlock() - if had { - node.Models = cached - node.ModelsByEngine = cachedByEngine - node.LoadedByEngine = cachedLoaded - } -} - -// fetchModels does one GET of a node's /v1/models, returning the -// flat model-name union, the per-engine attribution (engine name -> models), -// the per-engine loaded (in-memory) set, and whether it succeeded. Shape: +// fetchModels reads /v1/models and retains the server certificate so a remote +// result can be rechecked before updating the directory or caches. Response shape: // {"models":["name",...],"modelsByEngine":{"ollama":[...]},"loadedByEngine":{"ollama":[...]}}. // modelsByEngine and loadedByEngine are additive (omitempty on the wire); a peer // that doesn't send them yields nil maps, and the flat union still enriches the // node. A present engine key with [] is a successful empty inventory. -// A peer's model inventory is cluster data, so a peer is fetched over cluster -// mTLS keyed on the cluster principal it advertises (clusterUUID, from its -// cluster-uuid= TXT): this node must hold a pin for that principal, or there is -// nothing to fetch and nothing is attempted. THIS node's own engine-manager is -// dialed over loopback (both callers force that for self), which stays plain and -// therefore keeps working when this node belongs to no cluster — a standalone -// machine still has to show its own models. -func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, map[string][]string, map[string][]string, bool) { +// Peers use pinned mTLS and their verified clusterUUID. The local engine-manager +// uses loopback HTTP, including outside a cluster. +func (d *daemon) fetchModels(ip string, port int, clusterUUID string) (modelInventory, bool) { client, scheme, ok := d.modelsClient(ip, clusterUUID) if !ok { slog.Debug("model enrichment skipped: peer is not a pinned cluster member", "ip", ip, "clusterUuid", clusterUUID) - return nil, nil, nil, false + return modelInventory{}, false } url := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port)) + "/v1/models" resp, err := client.Get(url) if err != nil { slog.Debug("model enrichment failed", "ip", ip, "url", url, "err", err) - return nil, nil, nil, false + return modelInventory{}, false } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, nil, nil, false + return modelInventory{}, false } var body struct { Models []string `json:"models"` @@ -1432,9 +1266,17 @@ func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, LoadedByEngine map[string][]string `json:"loadedByEngine"` } if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - return nil, nil, nil, false + return modelInventory{}, false } - return body.Models, body.ModelsByEngine, body.LoadedByEngine, true + inventory := modelInventory{models: body.Models, byEngine: body.ModelsByEngine, loadedByEngine: body.LoadedByEngine} + if scheme == "https" { + d.mesh.Refresh() + if resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 || !d.mesh.VerifyPeerCertificate(clusterUUID, resp.TLS.PeerCertificates[0]) { + return modelInventory{}, false + } + inventory.certificate = resp.TLS.PeerCertificates[0] + } + return inventory, true } // modelsClient picks the transport for a model fetch: the plain client over @@ -1443,7 +1285,7 @@ func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, // false when a peer holds no pin here, which is the client-side cluster gate — // there is deliberately no plaintext fallback for a peer. func (d *daemon) modelsClient(ip, clusterUUID string) (*http.Client, string, bool) { - if isLoopbackHost(ip) { + if clusterUUID == "" && isLoopbackHost(ip) { return d.modelsHTTP, "http", true } d.mesh.Refresh() @@ -1463,17 +1305,9 @@ func isLoopbackHost(host string) bool { return ip != nil && ip.IsLoopback() } -// refreshPeersLoop periodically re-reads what each peer reports about itself over -// HTTP and folds any change back into the directory, independent of mDNS browse -// events. It is the convergence mechanism for facts that change without moving -// the raw mDNS record, or whose announcement was simply not received — neither of -// which produces a browse event, so neither reaches the enrichment in onBrowse. -// Stops when ctx is cancelled (the daemon's existing shutdown path). -// -// Cluster membership is refreshed before models on the same tick, because the -// model fetch is pinned mTLS keyed on the peer's principal: reading membership -// first means a peer that just joined or left is dialed under its current -// identity rather than the previous one. +// refreshPeersLoop refreshes models and local address selection periodically. +// Local publications and newly authenticated peers wake models immediately. Sweeps +// finish before the next starts, sharing the same model concurrency limit. func (d *daemon) refreshPeersLoop(ctx context.Context) { ticker := time.NewTicker(peerRefreshInterval) defer ticker.Stop() @@ -1481,13 +1315,14 @@ func (d *daemon) refreshPeersLoop(ctx context.Context) { select { case <-ctx.Done(): return + case <-d.modelsWake: + d.refreshModelsOnce() case <-ticker.C: d.refreshAdvertisedAddressesOnce() d.mesh.Refresh() if d.modelsClients != nil { d.modelsClients.DropUnpinned() } - d.refreshClusterIdentityOnce() d.refreshModelsOnce() } } @@ -1563,7 +1398,7 @@ func (d *daemon) dispatchTelemetry( if !ok { continue } - if n.HostUUID == self { + if n.HostUUID == self || n.IdentityVerification != noderec.IdentityPinned { continue // local telemetry arrives through discovery:local-node-info } hosts := append([]string(nil), n.CandidateIPs()...) @@ -1595,181 +1430,62 @@ func (d *daemon) dispatchTelemetry( } } -const maxTelemetryAge = int64(1<<63 - 1) - -// refreshNodeTelemetry emits one node's current GPU utilization. Failed fetches -// stay silent so downstream freshness expiry handles temporary unreachability. -// A response naming another host is never attributed to the directory entry -// whose address was queried. -func (d *daemon) refreshNodeTelemetry(ctx context.Context, hostUUID, ip string, niPort int) bool { - return d.refreshNodeTelemetryCandidates(ctx, hostUUID, []string{ip}, niPort) -} - -// telemetrySample is one node-info answer plus when the request that produced it -// began. The reported age has to be advanced by that request's own latency, and a -// failover attempt asks several addresses at once, so the time belongs to the -// answer rather than to the sweep. -type telemetrySample struct { - info NodeInfoResponse - startedAt time.Time -} - -// refreshNodeTelemetryCandidates is refreshNodeTelemetry with address failover. -// -// Healthy nodes are sampled every two seconds. Failed remote nodes back off, and -// each due attempt goes through the same node-info memory the enrichment sweeps -// use: the address that answered is asked by itself, and the rest of the list -// only once it stops answering. Without that, a peer whose canonical address is -// a link this host cannot reach reported no telemetry at all, and the scheduler -// then assigned it neutral pressure and scheduled it blind. +// refreshNodeTelemetryCandidates reads the current canonical peer target once +// for inventory and scheduling. Failed reads retain inventory while samples age; +// a retained peer can expire after its discoveries and availability evidence end. func (d *daemon) refreshNodeTelemetryCandidates(ctx context.Context, hostUUID string, hosts []string, niPort int) bool { - ask := func(host string) (telemetrySample, bool) { - startedAt := time.Now() - candidate, ok := d.fetchNodeInfoWithin(ctx, host, niPort) - if !ok { - return telemetrySample{}, false - } - if candidate.HostUUID != hostUUID { - slog.Debug("skipping telemetry reported by a different host at this address; trying next", - "expected_host_uuid", hostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host) - return telemetrySample{}, false - } - return telemetrySample{info: candidate, startedAt: startedAt}, true - } - key := hostKey{hostUUID: hostUUID, service: noderec.ServiceNodeInfo} - _, sample, ok := askRemembered(&d.enrichHosts, key, hosts, ask) - if !ok { + node, ok := d.dir.get(hostUUID) + if !ok || node.IdentityVerification != noderec.IdentityPinned || node.Services[noderec.ServiceNodeInfo].Port != niPort || !slices.Equal(node.CandidateIPs(), hosts) { return false } - info, startedAt := sample.info, sample.startedAt - - age := int64(0) - if info.TelemetryValid { - age = info.MSSince - if age < 0 { - age = 0 - } - elapsed := time.Since(startedAt).Milliseconds() - if elapsed > maxTelemetryAge-age { - age = maxTelemetryAge - } else { - age += elapsed + sample, ok := d.readPeerInfo(ctx, node) + if !ok { + if ctx.Err() == nil { + d.noteNodeInfo(hostUUID, net.JoinHostPort(node.IP, strconv.Itoa(niPort)), false) + d.expireRetainedPeer(node) } + return false } - var utilization uint32 - for i := range info.GPUs { - if info.GPUs[i].UtilizationPercent > utilization { - utilization = info.GPUs[i].UtilizationPercent - } + d.peerMu.Lock() + defer d.peerMu.Unlock() + if !d.currentPeerResult(node, sample) { + return false } - d.emitTelemetry(noderec.NodeTelemetry{ - HostUUID: hostUUID, - GPUUtilizationPct: utilization, - TelemetryValid: info.TelemetryValid, - MSSince: age, - }) + d.noteNodeInfo(hostUUID, net.JoinHostPort(node.IP, strconv.Itoa(niPort)), true) + d.publishPeerInfo("", sample) return true } -// refreshClusterIdentityOnce re-reads every peer's cluster membership from its -// node-info endpoint and folds any change into the directory. -// -// Membership otherwise reaches this daemon only as the cluster-uuid= key on a -// peer's mDNS record, which is read exactly once per browse *change* event. A -// peer that leaves its cluster and re-advertises produces such an event — but if -// it is not received, nothing asks again: the browse diff has no notion of a fact -// it never saw, and the liveness probe keeps the entry alive precisely because the -// peer is still reachable. The stale principal then outlives the cluster, and a -// client suppresses the invite that would bring that peer back. -// -// node-info is the right place to ask because it is the one inter-node surface -// deliberately kept plain (see the broker's spawnNodeInfo), so it answers whether -// or not this node and the peer share a cluster — and it is reached over TCP at -// the address the peer advertised, sharing nothing with multicast reception. -// -// Self is excluded: this node's own entry is registry-driven (publishSelf owns its -// principal), not something to learn from a loopback HTTP read. -func (d *daemon) refreshClusterIdentityOnce() { - self := "" - if d.reg != nil { - self = d.reg.record().HostUUID - } - // One refresh for the whole sweep: every peer is then annotated against the - // same pin set, and 16 concurrent re-reads of the trust store are avoided. - d.mesh.Refresh() - sem := make(chan struct{}, modelsRefreshConcurrency) - var wg sync.WaitGroup - for _, n := range d.dir.snapshot(noderec.ServiceNodeInfo) { - ni, ok := n.Services[noderec.ServiceNodeInfo] - if !ok || n.IP == "" || n.HostUUID == self { - continue +// expireRetainedPeer removes an unavailable peer after its last discovery has +// expired. The caller has just failed to refresh target. Membership and pins stay. +func (d *daemon) expireRetainedPeer(target noderec.DirectoryNode) { + d.peerMu.Lock() + defer d.peerMu.Unlock() + for _, candidate := range d.candidates { + if candidate.peerUUID == target.HostUUID { + return } - wg.Add(1) - sem <- struct{}{} - hosts := append([]string(nil), n.CandidateIPs()...) - go func(hostUUID string, hosts []string, port int) { - defer wg.Done() - defer func() { <-sem }() - d.refreshNodeClusterIdentityCandidates(hostUUID, hosts, port) - }(n.HostUUID, hosts, ni.Port) } - wg.Wait() -} - -// refreshNodeClusterIdentity reads one peer's reported membership and applies it, -// emitting one discovery:node-updated when it moved. Returns whether it emitted -// (used by tests). -// -// Two answers mean "leave the annotation alone": a failed fetch, and a peer that -// reports no clusterUuid field at all because it predates it. Only the peer's own -// explicit value is acted on — inferring "unclustered" from silence would mark a -// clustered peer invitable, which is the failure this annotation exists to -// prevent. -// -// A third: the machine answering may not be the one this entry describes. An -// address outlives its occupant — a wiped and reinstalled node, or a reused DHCP -// lease — and node-info names who it really is, so a mismatched hostUuid means -// writing a stranger's membership onto someone else's entry. Skipped rather than -// corrected here: the entry ages out through the same identity check in the -// liveness probe (see identityAlive), which owns that decision. A peer reporting -// no hostUuid predates the field and cannot be checked either way, so it is taken -// at its word — the same fallback that probe applies to the same answer. -func (d *daemon) refreshNodeClusterIdentity(hostUUID, ip string, niPort int) bool { - return d.refreshNodeClusterIdentityCandidates(hostUUID, []string{ip}, niPort) -} - -// refreshNodeClusterIdentityCandidates is refreshNodeClusterIdentity with -// address failover. A response from a different host is not authoritative for -// this entry, but it is also not the end of the candidate list: shared -// direct-connect addresses can legitimately resolve to a different machine. -func (d *daemon) refreshNodeClusterIdentityCandidates(hostUUID string, hosts []string, niPort int) bool { - ask := func(host string) (NodeInfoResponse, bool) { - candidate, ok := d.fetchNodeInfo(host, niPort) - if !ok { - return NodeInfoResponse{}, false - } - if candidate.HostUUID != "" && candidate.HostUUID != hostUUID { - slog.Debug("skipping cluster membership reported by a different host at this address; trying next", - "expected_host_uuid", hostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host) - return NodeInfoResponse{}, false - } - return candidate, true + peer, present := d.dir.get(target.HostUUID) + if !present || peer.IdentityVerification != noderec.IdentityPinned || + peer.DiscoveryRecordID != target.DiscoveryRecordID || peer.ClusterUUID != target.ClusterUUID || + peer.Services[noderec.ServiceNodeInfo].Port != target.Services[noderec.ServiceNodeInfo].Port || + !slices.Equal(peer.CandidateIPs(), target.CandidateIPs()) { + return } - // Shares the node-info memory with the enrichment sweep, so the two node-info - // callers converge on one address per peer rather than each finding it again. - key := hostKey{hostUUID: hostUUID, service: noderec.ServiceNodeInfo} - _, info, _ := askRemembered(&d.enrichHosts, key, hosts, ask) - if info.ClusterUUID == nil { - return false + d.infoMu.Lock() + at := d.lastInfoAt[peer.HostUUID] + d.infoMu.Unlock() + if time.Since(at) < retainedPeerGrace { + return } - node, changed := d.dir.applyClusterIdentity(hostUUID, info.ClusterUUID, d.mesh.HasPin) - if !changed { - return false + if since, ok := d.activitySince(peer.HostUUID); ok && since < activityFreshness { + return } - slog.Info("peer cluster membership changed", "host_uuid", hostUUID, - "clustered", node.Clustered(), "trusted", node.Trusted) - d.emit(noderec.NotifyNodeUpdated, node) - return true + d.dir.remove(peer.HostUUID) + delete(d.peerCertificates, peer.HostUUID) + d.forget(peer.HostUUID) + d.emit(noderec.NotifyNodeRemoved, peer) } // refreshModelsOnce snapshots every directory node advertising engine-manager @@ -1784,7 +1500,7 @@ func (d *daemon) refreshModelsOnce() { var wg sync.WaitGroup for _, n := range d.dir.snapshot(noderec.ServiceEngineManager) { em, ok := n.Services[noderec.ServiceEngineManager] - if !ok || n.IP == "" { + if !ok || n.IP == "" || (n.IdentityVerification != noderec.IdentityLocal && n.IdentityVerification != noderec.IdentityPinned) { continue } wg.Add(1) @@ -1816,26 +1532,26 @@ func (d *daemon) refreshModelsOnce() { // stale-address guard, so an in-flight result can't overwrite a re-addressed // node. Peers are still dialed at their advertised address. func (d *daemon) refreshNodeModels(hostUUID, ip string, emPort int, clusterUUID string) bool { - return d.refreshNodeModelsCandidates(hostUUID, ip, []string{ip}, emPort, clusterUUID) + node, present := d.dir.get(hostUUID) + if !present { + return false + } + return d.refreshNodeModelsCandidates(hostUUID, ip, append([]string(nil), node.CandidateIPs()...), emPort, clusterUUID) } // refreshNodeModelsCandidates is refreshNodeModels with address failover. The -// canonical IP remains the stale-result guard even when a later address is the -// one that answered. +// complete remote address list guards against queued or in-flight results from +// an endpoint removed by a newer authenticated inventory response. // -// Which address is asked is askRemembered's business — the one that answered last -// sweep, alone — and it shares that memory with the enrichment path, so both -// converge on one address per peer instead of each rediscovering it. +// askRemembered retains a working address across model sweeps until it fails or +// is withdrawn from the current record. func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []string, emPort int, clusterUUID string) bool { if d.reg != nil && hostUUID == d.reg.record().HostUUID { hosts = []string{loopbackHost} + clusterUUID = "" } ask := func(host string) (modelInventory, bool) { - models, byEngine, loadedByEngine, fetched := d.fetchModels(host, emPort, clusterUUID) - if !fetched { - return modelInventory{}, false - } - return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true + return d.fetchModels(host, emPort, clusterUUID) } key := hostKey{hostUUID: hostUUID, service: noderec.ServiceEngineManager} _, inventory, fetched := askRemembered(&d.enrichHosts, key, hosts, ask) @@ -1843,33 +1559,38 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s return false } models, byEngine, loadedByEngine := inventory.models, inventory.byEngine, inventory.loadedByEngine - // Refresh the enrichment cache BEFORE touching the directory: the caches - // (infoMu) and the directory (dir.mu) are guarded by different locks, so if - // the directory were updated first, a concurrent onBrowse->enrichModelsAt whose - // own fetch fails would fall back to a still-stale lastModels and upsert the - // old inventory, reverting this update. Writing the cache first means that - // fallback reads the fresh value. The only side effect — caching a value for - // a node applyModels then finds removed — is benign (it is the last-known-good - // value the cache exists to preserve, and forget() clears it on eviction). - d.infoMu.Lock() - if d.lastModels == nil { - d.lastModels = make(map[string][]string) - } - if d.lastModelsByEngine == nil { - d.lastModelsByEngine = make(map[string]map[string][]string) - } - if d.lastLoadedByEngine == nil { - d.lastLoadedByEngine = make(map[string]map[string][]string) + d.peerMu.Lock() + defer d.peerMu.Unlock() + d.mesh.Refresh() + if target, present := d.dir.get(hostUUID); present && target.IdentityVerification == noderec.IdentityPinned { + if target.ClusterUUID != clusterUUID || + !slices.Equal(target.CandidateIPs(), hosts) || + !d.mesh.VerifyPeerCertificate(clusterUUID, inventory.certificate) || + !d.mesh.VerifyPeerCertificate(clusterUUID, d.peerCertificates[hostUUID]) { + return false + } } - d.lastModels[hostUUID] = models - d.lastModelsByEngine[hostUUID] = byEngine - d.lastLoadedByEngine[hostUUID] = loadedByEngine - d.infoMu.Unlock() - node, changed, valid := d.dir.applyModels(hostUUID, guardIP, emPort, models, byEngine, loadedByEngine) if !valid { return false } + if node.IdentityVerification == noderec.IdentityLocal { + d.infoMu.Lock() + if d.lastModels == nil { + d.lastModels = make(map[string][]string) + } + if d.lastModelsByEngine == nil { + d.lastModelsByEngine = make(map[string]map[string][]string) + } + if d.lastLoadedByEngine == nil { + d.lastLoadedByEngine = make(map[string]map[string][]string) + } + d.lastModels[hostUUID] = models + d.lastModelsByEngine[hostUUID] = byEngine + d.lastLoadedByEngine[hostUUID] = loadedByEngine + d.infoMu.Unlock() + } + if changed { d.emit(noderec.NotifyNodeUpdated, node) } @@ -1880,12 +1601,10 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s // re-discovery re-probes from scratch rather than resurrecting stale metrics. func (d *daemon) forget(hostUUID string) { d.telemetryRetries.forget(hostUUID) + d.enrichHosts.forget(hostKey{hostUUID: "peer:" + hostUUID, service: noderec.ServiceNodeInfo}) + d.enrichHosts.forget(hostKey{hostUUID: hostUUID, service: noderec.ServiceEngineManager}) d.infoMu.Lock() - delete(d.lastInfo, hostUUID) delete(d.lastInfoAt, hostUUID) - delete(d.lastModels, hostUUID) - delete(d.lastModelsByEngine, hostUUID) - delete(d.lastLoadedByEngine, hostUUID) delete(d.nodeInfoDown, hostUUID) d.infoMu.Unlock() d.activityMu.Lock() diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index ddbf98eb3..1653262d5 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -6,15 +6,21 @@ package main import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" + "io" "net" "net/http" "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" + "runtime" "strconv" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -23,11 +29,76 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" ) -// selfAddrs turns one test address into the ranked candidate list the registry -// holds, mapping "" to "this host has no publishable address". +func TestDiscovery_CandidateAliasesYieldBusyOriginToVerifiedTelemetry(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + var requests atomic.Int32 + f.respond = func(http.ResponseWriter, *http.Request) bool { + requests.Add(1) + return false + } + + // Hold an existing read's permit so copied announcements cannot line up + // ahead of the verified peer's next scheduling sample. + release, acquired := f.daemon.nodeInfoOrigins.acquire(context.Background(), f.server.URL) + require.True(t, acquired) + var releaseOnce sync.Once + ctx, cancel := context.WithCancel(context.Background()) + authDone := make(chan struct{}) + t.Cleanup(func() { + cancel() + releaseOnce.Do(release) + <-authDone + }) + for i := range 4 { + alias := f.raw + alias.ID = fmt.Sprintf("copied peer %d", i) + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: alias}) + } + go func() { + f.daemon.refreshCandidatesOnce(ctx) + close(authDone) + }() + + select { + case <-authDone: + case <-time.After(time.Second): + require.FailNow(t, "candidate aliases queued behind the busy verified endpoint") + } + assert.Zero(t, requests.Load()) + nodes := f.daemon.dir.snapshot("") + require.Len(t, nodes, 5, "the verified peer and four separate aliases must remain") + var unverified int + for _, node := range nodes { + if node.IdentityVerification == noderec.IdentityUnverified { + assert.Equal(t, "peer-uuid", node.HostUUID) + assert.False(t, node.Trusted) + unverified++ + } + } + assert.Equal(t, 4, unverified) + + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + releaseOnce.Do(release) + f.daemon.refreshTelemetryOnce(ctx) + + message, err := NewCodec(&output).Read() + require.NoError(t, err) + require.Equal(t, noderec.NotifyNodeTelemetry, message.Method) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, "peer-uuid", sample.HostUUID) + assert.Equal(t, uint32(fakePeerGPUUtilization), sample.GPUUtilizationPct) + assert.True(t, sample.TelemetryValid) + assert.Equal(t, int32(1), requests.Load()) +} + +// selfAddrs returns the test address as a list, or nil when no address is available. func selfAddrs(ip string) []string { if ip == "" { return nil @@ -35,50 +106,45 @@ func selfAddrs(ip string) []string { return []string{ip} } -// newSelfTestDaemon builds a daemon wired for the registry-driven self path: a -// registry stamped with hostUUID/ip, a real directory, a disabled mesh, and -// initialized enrichment caches + HTTP clients. No codec/responder (emit is -// nil-safe; publishSelf skips re-advertise when the responder is nil). +// newSelfTestDaemon creates a scanner with a local UUID, address, and service +// registry. Cluster membership is disabled, caches and HTTP clients are ready, +// and no network announcements or IPC notifications are sent. func newSelfTestDaemon(hostUUID, ip string) *daemon { return &daemon{ mesh: clustertrust.Open(""), reg: newRegistry(hostUUID, "", selfAddrs(ip)), instance: "myhost", dir: newDirectory(), - http: &http.Client{Timeout: nodeInfoFetchTimeout}, modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, - lastInfo: make(map[string]NodeInfoResponse), lastInfoAt: make(map[string]time.Time), lastModels: make(map[string][]string), lastModelsByEngine: make(map[string]map[string][]string), lastLoadedByEngine: make(map[string]map[string][]string), + lastActivityAt: make(map[string]time.Time), + authWake: make(chan struct{}, 1), + modelsWake: make(chan struct{}, 1), } } -// TestReloadIdentityAdoptsClusterPrincipal covers the fresh-host uuid= re-stamp: -// the scanner spawns first and mints its own node-id, then cluster-manager -// writes identity.json; reloadIdentity must re-resolve and adopt that principal -// so the advertised uuid= converges (no lingering scanner-minted id) — AND move -// this node's own directory entry to the new uuid so the stale one isn't served -// to peers as a ghost. -func TestReloadIdentity_AdoptsClusterPrincipal(t *testing.T) { +// The scanner may create a UUID before cluster-manager writes identity.json. +// Reloading must use the UUID from that file and remove the old local record. +func TestReloadIdentity_UsesClusterIdentityUUIDAndRemovesOldRecord(t *testing.T) { base := t.TempDir() d := &daemon{ - mesh: clustertrust.Open(""), // unclustered: no cluster dir -> no clusterUuid + mesh: clustertrust.Open(""), // No cluster directory means no cluster membership. reg: newRegistry("scanner-minted-X", "", selfAddrs("127.0.0.1")), dir: newDirectory(), baseDir: base, - lastInfo: make(map[string]NodeInfoResponse), lastInfoAt: make(map[string]time.Time), lastModels: make(map[string][]string), lastModelsByEngine: make(map[string]map[string][]string), lastLoadedByEngine: make(map[string]map[string][]string), - // codec/responder nil: emit is nil-safe; reloadIdentity skips re-advertise. + // Leave codec and responder nil to suppress notifications and announcements. } - // Simulate this node already having been browsed under its scanner-minted uuid. + // The local record initially uses the UUID created by the scanner. d.dir.upsert(noderec.DirectoryNode{HostUUID: "scanner-minted-X", Name: "host", IP: "127.0.0.1"}) - // cluster-manager writes its principal under /cluster/identity.json. + // cluster-manager writes the UUID to /cluster/identity.json. idPath := filepath.Join(base, "cluster", "identity.json") require.NoError(t, os.MkdirAll(filepath.Dir(idPath), 0o700), "mkdir") body, err := json.Marshal(map[string]any{"node_uuid": "cluster-principal-Y", "created_at": 1}) @@ -87,49 +153,39 @@ func TestReloadIdentity_AdoptsClusterPrincipal(t *testing.T) { d.reloadIdentity() - require.Equal(t, "cluster-principal-Y", d.reg.record().HostUUID, "reloadIdentity hostUuid") - // Inbound reconciliation: the directory entry moved to the new uuid and the - // old one is gone (no ghost). - _, ok := d.dir.get("cluster-principal-Y") - assert.True(t, ok, "directory missing the re-stamped hostUuid after reloadIdentity") - _, ok = d.dir.get("scanner-minted-X") - assert.False(t, ok, "stale scanner-minted hostUuid left in the directory (ghost)") -} - -// TestSelfNotEvictedByBrowse is the regression: the local node must -// never be removed from its own directory by a browse event. On Windows the host -// frequently stops looping its own multicast back, so the browser ages self like -// any peer and (its LAN-IP liveness probe failing behind a first-run firewall) -// evicts it. With self registry-driven and ignored in onBrowse, a removed/ -// discovered/updated browse event for our own uuid is a no-op on the directory. -func TestBrowse_DoesNotEvictLocalNode(t *testing.T) { + assert.Equal(t, "cluster-principal-Y", d.reg.record().HostUUID) + // The directory must contain the new UUID and no longer contain the old one. + _, present := d.dir.get("cluster-principal-Y") + assert.True(t, present) + _, present = d.dir.get("scanner-minted-X") + assert.False(t, present) +} + +// Windows may stop receiving its own multicast announcements and report the local +// node as removed. Discovery events must preserve its local service registry record. +func TestDiscovery_BrowseEventsKeepLocalNode(t *testing.T) { d := newSelfTestDaemon("self-uuid", "192.168.1.17") d.reg.register(noderec.RegisterParams{Service: noderec.ServiceNodeInfo, Port: 14318}) d.publishSelf() _, ok := d.dir.get("self-uuid") require.True(t, ok, "publishSelf should put the local node in the directory") - // Its own record aged out of the browse (Windows self-multicast loss): the - // browser emits removed for our uuid. It must NOT drop the self entry. + // Losing the local announcement must not remove the local directory record. selfTXT := []string{"v=1", "uuid=self-uuid", "ip=192.168.1.17", "ni=14318"} d.onBrowse(DiscoveryEvent{Type: "removed", Node: RawNode{ID: "myhost", TXT: selfTXT}}) _, ok = d.dir.get("self-uuid") - assert.True(t, ok, "a browse 'removed' for our own uuid must not evict the local node") + assert.True(t, ok, "a removed event for our own UUID must preserve the local node") - // A later self re-appearance in the browse is also a no-op (self stays - // registry-driven; the browse never clobbers it). + // Receiving the announcement again must not replace the registry's local name. d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "myhost", TXT: selfTXT}}) - if n, ok := d.dir.get("self-uuid"); !ok { - assert.Fail(t, "self entry disappeared after a self 'discovered' browse event") - } else { - assert.Equal(t, "myhost", n.Name, "self entry Name") - } + n, ok := d.dir.get("self-uuid") + require.True(t, ok) + assert.Equal(t, "myhost", n.Name) } -func TestLocalNodeInfo_AttachesOwnedMeasurements(t *testing.T) { - // Verify that private worker samples populate the local registry record and - // scheduling telemetry before membership and after departure. Re-publishing - // local service registrations must retain that owned inventory. +func TestLocalNodeInfo_UsesMeasurementsFromLocalWorker(t *testing.T) { + // Use measurements from the local worker before joining and after leaving a + // cluster. Registering another service must retain those hardware measurements. want := NodeInfoResponse{ HostUUID: "self-uuid", GPUs: []GPUInfo{{Name: "NVIDIA GeForce RTX 5090"}}, @@ -162,7 +218,7 @@ func TestLocalNodeInfo_AttachesOwnedMeasurements(t *testing.T) { assert.Equal(t, want.GPUs, node.GPUs) assert.Equal(t, want.CPU, node.CPU) assert.Equal(t, want.Memory, node.Memory) - // A later registration must retain the owned hardware sample. + // A later service registration must retain the local hardware measurements. d.reg.register(noderec.RegisterParams{Service: noderec.ServiceOllama, Port: 11434}) d.publishSelf() node, _ = d.dir.get("self-uuid") @@ -186,6 +242,29 @@ func TestLocalNodeInfo_AttachesOwnedMeasurements(t *testing.T) { test("after leaving", true) } +// Local scheduling must reflect the busiest GPU, regardless of its list position. +func TestLocalNodeInfo_ReportsHighestGPUUtilization(t *testing.T) { + d := newSelfTestDaemon("self-uuid", "192.0.2.17") + params, err := json.Marshal(noderec.LocalNodeInfoParams{Info: noderec.NodeInfo{ + HostUUID: "self-uuid", TelemetryValid: true, + GPUs: []GPUInfo{ + {Name: "first GPU", UtilizationPercent: 12}, + {Name: "busiest GPU", UtilizationPercent: 84}, + {Name: "last GPU", UtilizationPercent: 20}, + }, + }}) + require.NoError(t, err) + var output bytes.Buffer + d.codec = NewCodec(&output) + + require.True(t, d.handle(&Message{Method: noderec.MethodLocalNodeInfo, Params: params})) + + message := readTelemetryMessage(t, json.NewDecoder(&output)) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, uint32(84), sample.GPUUtilizationPct) +} + func TestLocalNodeInfo_NeverPollsPublicInventory(t *testing.T) { // Provide a public endpoint that would answer local reads, then verify that // local publication and telemetry sampling issue no requests to it. @@ -206,10 +285,8 @@ func TestLocalNodeInfo_NeverPollsPublicInventory(t *testing.T) { assert.Equal(t, int32(0), requests.Load()) } -// TestPublishSelfLoopbackIPFallback covers the "IP unknown at startup" case: when -// the LAN ranker hasn't picked an address, self still gets a dialable host -// (loopback) so the local card is never address-less. -func TestPublishSelf_UsesLoopbackWhenNoAddressIsPublished(t *testing.T) { +// Use the loopback address when no LAN address has been selected at startup. +func TestPublishSelf_UsesLoopbackWhenNoLANAddressIsAvailable(t *testing.T) { d := newSelfTestDaemon("self-uuid", "") // BestLocalIP() empty d.publishSelf() n, ok := d.dir.get("self-uuid") @@ -217,148 +294,1216 @@ func TestPublishSelf_UsesLoopbackWhenNoAddressIsPublished(t *testing.T) { assert.Equal(t, loopbackHost, n.IP, "self IP with no LAN address (%v)", loopbackHost) } -// TestPeerStillAgesOut confirms the fix is scoped to self: a genuine remote peer -// is still added on discovery and removed on a browse 'removed' event. The peer -// advertises no services so onBrowse does no enrichment dial. -func TestReachable_RemovesPeerAfterLivenessExpires(t *testing.T) { +// An unverified peer record is removed when its announcement disappears. +// With no advertised services, there is no node-info endpoint to authenticate. +func TestDiscovery_RemovesUnverifiedRecordWhenAnnouncementDisappears(t *testing.T) { d := newSelfTestDaemon("self-uuid", "192.168.1.17") peerTXT := []string{"v=1", "uuid=peer-uuid", "ip=192.168.1.99"} d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "peer", TXT: peerTXT}}) - _, ok := d.dir.get("peer-uuid") - require.True(t, ok, "a discovered peer should be added to the directory") + _, present := d.dir.get(nodeDiscoveryRecordID(RawNode{ID: "peer", TXT: peerTXT})) + require.True(t, present) d.onBrowse(DiscoveryEvent{Type: "removed", Node: RawNode{ID: "peer", TXT: peerTXT}}) - _, ok = d.dir.get("peer-uuid") - assert.False(t, ok, "a genuine remote peer must still age out when it leaves") + _, present = d.dir.get(nodeDiscoveryRecordID(RawNode{ID: "peer", TXT: peerTXT})) + assert.False(t, present) +} + +func TestReachable_RejectsAdvertisementsClaimingTheLocalHost(t *testing.T) { + // Reject a discovery claim to our UUID even when a fresh cached sample + // would otherwise make that UUID reachable, without any network probe. + const selfUUID = "self-uuid" + d := newSelfTestDaemon(selfUUID, "192.0.2.1") + d.lastInfoAt[selfUUID] = time.Now() + self := RawNode{ + TXT: []string{"uuid=" + selfUUID, "ip=192.0.2.1"}, + } + assert.False(t, d.reachable(self), "an advertisement claiming our UUID must be rejected despite a fresh cached sample") } -// TestReachableAntiFlap covers the daemon's liveness probe: prefer fresh -// lastInfo, else TCP to node-info / engine-manager, then identity when -// conclusive. Never treat ol/lm alone as proof of life. -func TestReachable_RetainsNodeAcrossTransientFailures(t *testing.T) { - niSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - http.NotFound(w, r) +func TestPublishSelf_LocalRecordHasLocalVerificationAndDiscoveryRecordID(t *testing.T) { + d := newSelfTestDaemon("self-uuid", "192.0.2.10") + d.publishSelf() + node, ok := d.dir.get("self-uuid") + require.True(t, ok, "local registry record must exist") + assert.Equal(t, noderec.IdentityLocal, node.IdentityVerification) + assert.Equal(t, "local:self-uuid", node.DiscoveryRecordID) +} + +type discoveryPeerFixture struct { + daemon *daemon + selfDir string + peerDir string + server *httptest.Server + raw RawNode + info NodeInfoResponse + respond func(http.ResponseWriter, *http.Request) bool + connections atomic.Int32 +} + +const ( + fakePeerPort = 11434 + fakePeerGPUUtilization = 84 + fakePeerSampleAgeMillis = 137 +) + +// newDiscoveryPeerFixture creates an isolated scanner and TLS peer. Each saves +// the other's certificate as its expected identity (certificate pinning). +// The returned fixture exposes the server, node-info response, and announcement. +func newDiscoveryPeerFixture(t *testing.T, uuid string) *discoveryPeerFixture { + t.Helper() + selfDir, peerDir := t.TempDir(), t.TempDir() + clustertrusttest.Join(t, selfDir, "test-cluster", "self-uuid") + clustertrusttest.Join(t, peerDir, "test-cluster", uuid) + writePinFromCert(t, selfDir, uuid, filepath.Join(peerDir, "node.crt")) + writePinFromCert(t, peerDir, "self-uuid", filepath.Join(selfDir, "node.crt")) + mesh, peerMesh := clustertrust.Open(selfDir), clustertrust.Open(peerDir) + f := &discoveryPeerFixture{selfDir: selfDir, peerDir: peerDir} + f.info = NodeInfoResponse{ + HostUUID: uuid, ClusterUUID: &uuid, TelemetryValid: true, MSSince: fakePeerSampleAgeMillis, + GPUs: []GPUInfo{{Name: "verified GPU", UtilizationPercent: fakePeerGPUUtilization}}, + Peer: &noderec.PeerMetadata{Name: "owned peer", Addresses: []string{"127.0.0.1"}}, + } + f.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peerMesh.Refresh() + if _, ok := peerMesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", http.StatusForbidden) return } - _, err := w.Write([]byte(`{}`)) - assert.NoError(t, err) + if f.respond != nil && f.respond(w, r) { + return + } + assert.NoError(t, json.NewEncoder(w).Encode(f.info)) })) - defer niSrv.Close() - niURL, err := url.Parse(niSrv.URL) - require.NoError(t, err, "parse ni url") - niPort, err := strconv.Atoi(niURL.Port()) - require.NoError(t, err, "ni port") - - emLn, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err, "em listen") - defer emLn.Close() - emPort := emLn.Addr().(*net.TCPAddr).Port - - olLn, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err, "ol listen") - defer olLn.Close() - olPort := olLn.Addr().(*net.TCPAddr).Port - - dl, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err, "dead listen") - deadPort := dl.Addr().(*net.TCPAddr).Port - require.NoError(t, dl.Close()) + f.server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + f.connections.Add(1) + } + } + f.server.TLS = &tls.Config{MinVersion: tls.VersionTLS12, GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + peerMesh.Refresh() + return peerMesh.ServerTLSConfig().GetConfigForClient(hello) + }} + f.server.StartTLS() + t.Cleanup(f.server.Close) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services = map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port, noderec.ServiceOllama: fakePeerPort} + f.raw = RawNode{ID: "unverified name", Host: "peer.local.", Addresses: []string{"127.0.0.1"}, + TXT: []string{"uuid=" + uuid, "cluster-uuid=" + uuid, fmt.Sprintf("ni=%d", port), "ol=1"}} + f.daemon = newSelfTestDaemon("self-uuid", "192.0.2.1") + f.daemon.mesh = mesh + f.daemon.clusterDir = selfDir + // Trust reload may resolve local identity. Keep that lookup inside this + // fixture rather than the developer's default application directory. + f.daemon.baseDir = t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(f.daemon.baseDir, "node-id.json"), []byte(`{"node_uuid":"self-uuid"}`), 0o600)) + f.daemon.modelsClients = clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout) + t.Cleanup(f.daemon.modelsClients.CloseIdle) + f.daemon.nodeInfoClients = newNodeInfoClients(mesh) + t.Cleanup(f.daemon.nodeInfoClients.CloseIdle) + return f +} - d := &daemon{ - lastInfo: make(map[string]NodeInfoResponse), - lastInfoAt: make(map[string]time.Time), +func TestDiscovery_ReplacesUnverifiedDiscoveryWithVerifiedPeer(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + node, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + assert.Equal(t, noderec.IdentityPinned, node.IdentityVerification) + assert.True(t, node.Trusted) + assert.NotEqual(t, nodeDiscoveryRecordID(f.raw), node.DiscoveryRecordID) + assert.Equal(t, "owned peer", node.Name) + assert.Equal(t, fakePeerPort, node.Services[noderec.ServiceOllama].Port) + assert.Equal(t, f.info.GPUs, node.GPUs) + assert.Len(t, f.daemon.dir.snapshot(""), 1) +} + +func TestDiscovery_FetchesModelsAfterAuthentication(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.info.Peer.Services[noderec.ServiceEngineManager] = portFromURL(t, f.server.URL) + type modelResponse struct { + Models []string `json:"models"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + } + response := modelResponse{ + Models: []string{"reported-model"}, + ModelsByEngine: map[string][]string{"ollama": {"reported-model"}}, + LoadedByEngine: map[string][]string{"ollama": {"reported-model"}}, + } + requested := make(chan struct{}, 1) + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + node, ok := f.daemon.dir.get("peer-uuid") + assert.True(t, ok, "models require an authenticated peer") + assert.Equal(t, noderec.IdentityPinned, node.IdentityVerification) + requested <- struct{}{} + assert.NoError(t, json.NewEncoder(w).Encode(response)) + return true } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + t.Cleanup(func() { cancel(); <-done }) + go func() { + defer close(done) + f.daemon.refreshPeersLoop(ctx) + }() + f.discover(t) + select { + case <-requested: + case <-time.After(2 * time.Second): + require.FailNow(t, "authentication did not trigger model inventory before the periodic refresh") + } + // The refresh finishes its model update before the loop observes cancellation. + cancel() + <-done + node, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + assert.Equal(t, response.Models, node.Models) + assert.Equal(t, response.ModelsByEngine, node.ModelsByEngine) + assert.Equal(t, response.LoadedByEngine, node.LoadedByEngine) +} - liveNI := RawNode{TXT: []string{"v=1", "uuid=host-a", "ip=127.0.0.1", fmt.Sprintf("ni=%d", niPort)}} - assert.True(t, d.reachable(liveNI), "node with live node-info should be reachable") +func TestDiscovery_SpoofedClaimsCannotChangeVerifiedPeers(t *testing.T) { + // clusterUuid is the peer's node UUID from its cluster identity. + test := func(name, hostClaim, clusterUUIDClaim string) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + before, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok, "established peer must exist") + var publicReads atomic.Int32 + rogue := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + publicReads.Add(1) + assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: hostClaim, ClusterUUID: &clusterUUIDClaim, TelemetryValid: true, GPUs: []GPUInfo{{Name: "forged idle GPU"}}})) + })) + t.Cleanup(rogue.Close) + raw := RawNode{ID: "owned peer", Addresses: []string{"127.0.0.1"}, TXT: []string{"uuid=" + hostClaim, fmt.Sprintf("ni=%d", portFromURL(t, rogue.URL)), "ol=2"}} + if clusterUUIDClaim != "" { + raw.TXT = append(raw.TXT, "cluster-uuid="+clusterUUIDClaim) + } + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: raw}) + f.daemon.refreshCandidatesOnce(context.Background()) + after, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.Equal(t, before, after, "rogue discovery must preserve the established peer") + var unverified int + for _, node := range f.daemon.dir.snapshot("") { + if node.DiscoveryRecordID == nodeDiscoveryRecordID(raw) { + unverified++ + assert.False(t, node.Trusted) + assert.Equal(t, noderec.IdentityUnverified, node.IdentityVerification) + assert.Empty(t, node.GPUs) + } + } + assert.Equal(t, 1, unverified) + assert.Equal(t, int32(0), publicReads.Load()) + f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: raw}) + after, present = f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.Equal(t, before, after, "rogue removal must preserve the established peer") + }) + } + test("copied host and cluster UUIDs", "peer-uuid", "peer-uuid") + test("copied host UUID without cluster UUID", "peer-uuid", "") + test("copied host UUID with another cluster UUID", "peer-uuid", "other-peer") + test("new host UUID with copied cluster UUID", "fresh-uuid", "peer-uuid") +} - // The node's first-ranked address can be unreachable from this observer even - // when a later published address reaches the same live service. A transient - // mDNS miss must not evict that live multi-homed node. - liveNIOnSecondAddress := RawNode{TXT: []string{ - "v=1", "uuid=host-multihomed", "ip=127.0.0.2", - "ips=127.0.0.2,127.0.0.1", fmt.Sprintf("ni=%d", niPort), - }} - assert.True(t, d.reachable(liveNIOnSecondAddress), "node reachable at its second published address should stay alive") +func TestDiscovery_RemovingSavedCertificateRemovesVerifiedPeer(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + clustertrusttest.RemovePeerPin(t, f.selfDir, "peer-uuid") + f.daemon.reloadTrust() + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) +} - deadNI := RawNode{TXT: []string{"v=1", "ip=127.0.0.1", fmt.Sprintf("ni=%d", deadPort)}} - assert.False(t, d.reachable(deadNI), "node whose only ni port is closed should be unreachable") +// discover reports the peer's announcement and waits for authentication attempts. +func (f *discoveryPeerFixture) discover(t *testing.T) { + t.Helper() + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: f.raw}) + f.daemon.refreshCandidatesOnce(context.Background()) +} - // Live ol alone must not keep the node (inference proxy is not a liveness signal). - olOnly := RawNode{TXT: []string{"v=1", "ip=127.0.0.1", fmt.Sprintf("ol=%d", olPort)}} - assert.False(t, d.reachable(olOnly), "node with only a live ol port must not be reachable") +// refreshCandidatesOnce starts pending authentication attempts with the normal +// concurrency limit, then waits for them to finish so tests can check the result. +func (d *daemon) refreshCandidatesOnce(ctx context.Context) { + sem := make(chan struct{}, telemetryRefreshConcurrency) + var workers sync.WaitGroup + d.dispatchCandidates(ctx, sem, &workers) + workers.Wait() +} - // Dead ni + live ol: still unreachable — ol is ignored. - mixed := RawNode{TXT: []string{"v=1", "ip=127.0.0.1", fmt.Sprintf("ni=%d", deadPort), fmt.Sprintf("ol=%d", olPort)}} - assert.False(t, d.reachable(mixed), "dead ni with live ol must not be reachable") +// rescheduleAuthenticationFailure checks that dispatch recorded its failure, +// then moves the retry deadline using the gate's explicit timestamps. Tests can +// examine pending and expired retries without relying on wall-clock delays. +func (f *discoveryPeerFixture) rescheduleAuthenticationFailure(t *testing.T, checkedAt, finishedAt time.Time) { + t.Helper() + id := nodeDiscoveryRecordID(f.raw) + candidate := f.daemon.candidates[id] + require.NotNil(t, candidate) + node := candidate.node + target := telemetryTargetKey(node.CandidateIPs(), node.Services[noderec.ServiceNodeInfo].Port) + node.ClusterUUID + _, due := f.daemon.authRetries.claim(id, target, checkedAt) + require.False(t, due, "failed authentication must establish backoff") + token, due := f.daemon.authRetries.claim(id, target, time.Now().Add(24*time.Hour)) + require.True(t, due, "completed authentication must release its active attempt") + f.daemon.authRetries.finish(id, token, false, finishedAt) +} + +// Repeated observations of the same failing endpoint must retain its retry deadline. +func TestAuthenticationDispatch_AnnouncementUpdatesPreserveBackoff(t *testing.T) { + test := func(name string, extraTXT []string) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + var reads atomic.Int32 + f.respond = func(w http.ResponseWriter, _ *http.Request) bool { + reads.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + startedAt := time.Now() + f.discover(t) + require.Equal(t, int32(1), reads.Load()) + f.rescheduleAuthenticationFailure(t, startedAt, time.Now().Add(time.Hour)) + f.daemon.refreshCandidatesOnce(context.Background()) + require.Equal(t, int32(1), reads.Load(), "repeated dispatch must wait for the retry deadline") - // em fallback when ni is absent. - emOnly := RawNode{TXT: []string{"v=1", "ip=127.0.0.1", fmt.Sprintf("em=%d", emPort)}} - assert.True(t, d.reachable(emOnly), "node with live em and no ni should be reachable") + updated := f.raw + updated.TXT = append(append([]string(nil), f.raw.TXT...), extraTXT...) + f.daemon.onBrowse(DiscoveryEvent{Type: "updated", Node: updated}) + f.daemon.refreshCandidatesOnce(context.Background()) - // Fresh lastInfo skips dialing. - d.lastInfo["host-cached"] = NodeInfoResponse{} - d.lastInfoAt["host-cached"] = time.Now() - cached := RawNode{TXT: []string{"v=1", "uuid=host-cached", "ip=127.0.0.1", fmt.Sprintf("ni=%d", deadPort)}} - assert.True(t, d.reachable(cached), "fresh lastInfo should keep the node without a successful probe") + assert.Equal(t, int32(1), reads.Load(), "an unchanged node-info endpoint must keep its backoff") + }) + } + test("same announcement", nil) + test("unrelated service changes", []string{"er=14319"}) +} - idOnly := RawNode{TXT: []string{"v=1", "ip=127.0.0.1"}} - assert.False(t, d.reachable(idOnly), "identity-only node (no ni/em) should be unreachable") +// A failed candidate must be reconsidered after backoff without another announcement. +func TestAuthenticationDispatch_RetriesCandidateAfterBackoff(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + var reads atomic.Int32 + f.respond = func(w http.ResponseWriter, _ *http.Request) bool { + reads.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + startedAt := time.Now() + f.discover(t) + require.Equal(t, int32(1), reads.Load()) + f.rescheduleAuthenticationFailure(t, startedAt, time.Now().Add(time.Hour)) + f.daemon.refreshCandidatesOnce(context.Background()) + require.Equal(t, int32(1), reads.Load(), "failed candidate retried before backoff") - noIP := RawNode{TXT: []string{"v=1", fmt.Sprintf("ni=%d", niPort)}} - assert.False(t, d.reachable(noIP), "node with no resolvable address should be unreachable") + f.rescheduleAuthenticationFailure(t, startedAt, time.Now().Add(-time.Hour)) + f.daemon.refreshCandidatesOnce(context.Background()) + assert.Equal(t, int32(2), reads.Load(), "candidate must retry after its deadline without another announcement") } -func TestEnrich_FallsBackToASecondPublishedAddress(t *testing.T) { - const hostUUID = "host-enrich-failover" - niSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{ - HostUUID: hostUUID, - GPUs: []GPUInfo{{Name: "GPU from reachable address"}}, - })) - })) - defer niSrv.Close() - niURL, err := url.Parse(niSrv.URL) - require.NoError(t, err, "parse node-info URL") - niPort, err := strconv.Atoi(niURL.Port()) - require.NoError(t, err, "parse node-info port") +// A failed old endpoint must not delay authentication of a newly announced endpoint. +func TestAuthenticationDispatch_ChangedEndpointCanAuthenticateDuringBackoff(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + oldURL, err := url.Parse(f.server.URL) + require.NoError(t, err) + var oldReads, newReads atomic.Int32 + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.Host == oldURL.Host { + oldReads.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + newReads.Add(1) + return false + } + startedAt := time.Now() + f.discover(t) + require.Equal(t, int32(1), oldReads.Load()) + f.rescheduleAuthenticationFailure(t, startedAt, time.Now().Add(time.Hour)) - d := &daemon{ - http: niSrv.Client(), - lastInfo: make(map[string]NodeInfoResponse), - lastInfoAt: make(map[string]time.Time), - nodeInfoDown: make(map[string]bool), - } - node := noderec.DirectoryNode{ - HostUUID: hostUUID, - IP: "127.0.0.2", - IPs: []string{"127.0.0.2", "127.0.0.1"}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: niPort}, - }, + replacement := httptest.NewUnstartedServer(f.server.Config.Handler) + replacement.TLS = f.server.TLS + replacement.StartTLS() + t.Cleanup(replacement.Close) + port := portFromURL(t, replacement.URL) + f.info.Peer.Services[noderec.ServiceNodeInfo] = port + updated := f.raw + updated.TXT = append([]string(nil), f.raw.TXT...) + for i, field := range updated.TXT { + if strings.HasPrefix(field, "ni=") { + updated.TXT[i] = fmt.Sprintf("ni=%d", port) + } } - d.enrich(&node) - require.Len(t, node.GPUs, 1) - require.Equal(t, "GPU from reachable address", node.GPUs[0].Name) + f.daemon.onBrowse(DiscoveryEvent{Type: "updated", Node: updated}) + f.daemon.refreshCandidatesOnce(context.Background()) + + assert.Equal(t, int32(1), oldReads.Load(), "old endpoint must remain backed off") + assert.Equal(t, int32(1), newReads.Load()) + node, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.Equal(t, noderec.IdentityPinned, node.IdentityVerification) + assert.Equal(t, port, node.Services[noderec.ServiceNodeInfo].Port) } -func TestReachable_RejectsAdvertisementsClaimingTheLocalHost(t *testing.T) { - // Reject a discovery claim to our UUID even when a fresh cached sample - // would otherwise make that UUID reachable, without any network probe. - const selfUUID = "self-uuid" - d := newSelfTestDaemon(selfUUID, "192.0.2.1") - d.lastInfoAt[selfUUID] = time.Now() - self := RawNode{ - TXT: []string{"uuid=" + selfUUID, "ip=192.0.2.1"}, +func closedPort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + _, port := splitHostPort(t, ln.Addr().String()) + require.NoError(t, ln.Close()) + return port +} + +func splitHostPort(t *testing.T, addr string) (string, int) { + t.Helper() + host, portStr, err := net.SplitHostPort(addr) + require.NoError(t, err, "address %q", addr) + port, err := strconv.Atoi(portStr) + require.NoError(t, err, "port %q", portStr) + return host, port +} + +func TestDiscovery_RejectsMissingOrInvalidPeerDetails(t *testing.T) { + test := func(name string, change func(*NodeInfoResponse)) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + change(&f.info) + f.discover(t) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) + nodes := f.daemon.dir.snapshot("") + require.Len(t, nodes, 1) + assert.False(t, nodes[0].Trusted) + assert.Empty(t, nodes[0].GPUs) + }) } - assert.False(t, d.reachable(self), "an advertisement claiming our UUID must be rejected despite a fresh cached sample") + test("another host UUID", func(info *NodeInfoResponse) { info.HostUUID = "other-peer" }) + test("another cluster UUID", func(info *NodeInfoResponse) { uuid := "other-peer"; info.ClusterUUID = &uuid }) + test("missing host identity", func(info *NodeInfoResponse) { info.HostUUID = "" }) + test("missing cluster UUID", func(info *NodeInfoResponse) { info.ClusterUUID = nil }) + test("missing peer metadata", func(info *NodeInfoResponse) { info.Peer = nil }) + test("invalid address", func(info *NodeInfoResponse) { info.Peer.Addresses = []string{"not-an-IP"} }) + test("missing name", func(info *NodeInfoResponse) { info.Peer.Name = "" }) + test("missing node-info port", func(info *NodeInfoResponse) { delete(info.Peer.Services, noderec.ServiceNodeInfo) }) + test("invalid service port", func(info *NodeInfoResponse) { info.Peer.Services[noderec.ServiceOllama] = 65536 }) } -func TestPublishSelf_LocalRecordHasLocalVerificationAndDiscoveryRecordID(t *testing.T) { - d := newSelfTestDaemon("self-uuid", "192.0.2.10") - d.publishSelf() - node, ok := d.dir.get("self-uuid") - require.True(t, ok, "local registry record must exist") - assert.Equal(t, noderec.IdentityLocal, node.IdentityVerification) - assert.Equal(t, "local:self-uuid", node.DiscoveryRecordID) +func TestDiscovery_RejectsCertificatesThatDoNotMatchExpectedPeer(t *testing.T) { + test := func(name, presentedUUID string) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + wrongDir := t.TempDir() + clustertrusttest.Join(t, wrongDir, "test-cluster", presentedUUID) + // A certificate saved for another peer cannot authenticate this peer. + writePinFromCert(t, f.selfDir, presentedUUID, filepath.Join(wrongDir, "node.crt")) + if presentedUUID == "peer-uuid" { + writePinFromCert(t, f.selfDir, "peer-uuid", filepath.Join(f.peerDir, "node.crt")) + } + wrongMesh := clustertrust.Open(wrongDir) + f.server.TLS.GetConfigForClient = wrongMesh.ServerTLSConfig().GetConfigForClient + f.discover(t) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) + }) + } + test("new certificate for same UUID does not match saved certificate", "peer-uuid") + test("saved certificate belongs to another peer", "other-peer") +} + +func TestDiscovery_DiscardsResultsInvalidatedDuringRead(t *testing.T) { + test := func(name string, invalidate func(*testing.T, *discoveryPeerFixture)) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.invalidateDuringRead(t, invalidate, func() { f.discover(t) }) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) + }) + } + test("saved peer certificate removed", func(t *testing.T, f *discoveryPeerFixture) { clustertrusttest.RemovePeerPin(t, f.selfDir, "peer-uuid") }) + test("saved peer certificate replaced", func(t *testing.T, f *discoveryPeerFixture) { clustertrusttest.WritePeerPin(t, f.selfDir, "peer-uuid") }) + test("local node leaves cluster", func(t *testing.T, f *discoveryPeerFixture) { + clustertrusttest.RemovePeerPin(t, f.selfDir, "peer-uuid") + require.NoError(t, os.Remove(filepath.Join(f.selfDir, "admission.json"))) + }) + test("local certificate replaced", func(t *testing.T, f *discoveryPeerFixture) { clustertrusttest.WriteKeypair(t, f.selfDir, "self-uuid") }) + test("discovery removed", func(t *testing.T, f *discoveryPeerFixture) { + f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: f.raw}) + }) +} + +// invalidateDuringRead holds the HTTP response while the test changes trust. +// File setup and its fatal assertions run on the test goroutine. +func (f *discoveryPeerFixture) invalidateDuringRead(t *testing.T, invalidate func(*testing.T, *discoveryPeerFixture), read func()) { + t.Helper() + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + f.respond = func(http.ResponseWriter, *http.Request) bool { + close(entered) + <-release + return false + } + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) + go func() { + defer close(done) + read() + }() + select { + case <-entered: + case <-time.After(2 * time.Second): + require.FailNow(t, "inventory request did not reach the server") + } + invalidate(t, f) + releaseOnce.Do(func() { close(release) }) + <-done +} + +func TestDiscovery_ReauthenticatesAfterSavedCertificateChanges(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + clustertrusttest.WriteKeypair(t, f.peerDir, "peer-uuid") + writePinFromCert(t, f.selfDir, "peer-uuid", filepath.Join(f.peerDir, "node.crt")) + f.daemon.reloadTrust() + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) + f.daemon.refreshCandidatesOnce(context.Background()) + node, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.True(t, node.Trusted) +} + +type heldModelResponseTransport struct { + http.RoundTripper + closed chan<- struct{} + release <-chan struct{} +} + +func (transport heldModelResponseTransport) RoundTrip(request *http.Request) (*http.Response, error) { + response, err := transport.RoundTripper.RoundTrip(request) + if err != nil { + return nil, err + } + response.Body = &heldModelResponseBody{ReadCloser: response.Body, closed: transport.closed, release: transport.release} + return response, nil +} + +type heldModelResponseBody struct { + io.ReadCloser + closed chan<- struct{} + release <-chan struct{} +} + +func (body *heldModelResponseBody) Close() error { + err := body.ReadCloser.Close() + close(body.closed) + <-body.release + return err +} + +func TestDiscovery_DiscardsModelResponseAfterSavedCertificateReplacement(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + type modelResponse struct { + Models []string `json:"models"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + } + response := modelResponse{ + Models: []string{"reported-model"}, + ModelsByEngine: map[string][]string{"ollama": {"reported-model"}}, + LoadedByEngine: map[string][]string{"ollama": {"reported-model"}}, + } + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(response)) + return true + } + f.discover(t) + client, ok := f.daemon.modelsClients.Client("peer-uuid") + require.True(t, ok, "authenticated models client must be available") + closed, release := make(chan struct{}), make(chan struct{}) + // fetchModels closes the body after its certificate check. Pause there so + // replacement happens after that check but before the directory update. + client.Transport = heldModelResponseTransport{RoundTripper: client.Transport, closed: closed, release: release} + changed := make(chan bool, 1) + done := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) + go func() { + defer close(done) + changed <- f.daemon.refreshNodeModels("peer-uuid", "127.0.0.1", port, "peer-uuid") + }() + select { + case <-closed: + case <-time.After(2 * time.Second): + require.FailNow(t, "model response did not reach the point before directory update") + } + clustertrusttest.WriteKeypair(t, f.peerDir, "peer-uuid") + writePinFromCert(t, f.selfDir, "peer-uuid", filepath.Join(f.peerDir, "node.crt")) + f.daemon.reloadTrust() + f.discover(t) + before, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + require.True(t, before.Trusted) + require.Empty(t, before.Models) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + releaseOnce.Do(func() { close(release) }) + <-done + assert.False(t, <-changed, "model response from the replaced certificate must not update the new peer") + after, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + assert.Equal(t, before, after) + assert.Empty(t, output.Bytes()) + require.True(t, f.daemon.refreshNodeModels("peer-uuid", "127.0.0.1", port, "peer-uuid")) + after, _ = f.daemon.dir.get("peer-uuid") + assert.Equal(t, response.Models, after.Models) + assert.Equal(t, response.ModelsByEngine, after.ModelsByEngine) + assert.Equal(t, response.LoadedByEngine, after.LoadedByEngine) +} + +func TestDiscovery_PeerActivityDoesNotKeepSpoofedDiscoveryReachable(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.daemon.noteActivity("peer-uuid", 0) + require.True(t, f.daemon.reachable(f.raw), "recent authenticated node-info must keep its peer reachable") + spoof := f.raw + spoof.ID = "spoof" + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: spoof}) + assert.False(t, f.daemon.reachable(spoof), "verified activity must not keep a spoofed discovery reachable") + f.server.Close() + f.daemon.infoMu.Lock() + f.daemon.lastInfoAt["peer-uuid"] = time.Now().Add(-reachableInfoFreshness - time.Second) + f.daemon.infoMu.Unlock() + assert.True(t, f.daemon.reachable(f.raw), "recent inference activity must keep the verified peer reachable") + clustertrusttest.RemovePeerPin(t, f.selfDir, "peer-uuid") + assert.False(t, f.daemon.reachable(f.raw), "activity must not retain a peer after its certificate is removed") +} + +func TestDiscovery_RefreshesOnlyAuthenticatedInventory(t *testing.T) { + const updatedPort = 11435 + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.info.Peer.Name = "updated owned name" + f.info.Peer.Services[noderec.ServiceOllama] = updatedPort + f.info.CPU = &CPUInfo{Name: "updated CPU"} + f.daemon.refreshTelemetryOnce(context.Background()) + node, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + assert.Equal(t, f.info.Peer.Name, node.Name) + assert.Equal(t, updatedPort, node.Services[noderec.ServiceOllama].Port) + assert.Equal(t, f.info.CPU, node.CPU) + f.server.Close() + f.daemon.refreshTelemetryOnce(context.Background()) + after, _ := f.daemon.dir.get("peer-uuid") + assert.Equal(t, node, after, "temporary failure must preserve inventory") +} + +func TestDiscovery_SlowAuthenticationDoesNotBlockVerifiedPeerTelemetry(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + slow := newDiscoveryPeerFixture(t, "slow-peer") + writePinFromCert(t, f.selfDir, "slow-peer", filepath.Join(slow.peerDir, "node.crt")) + writePinFromCert(t, slow.peerDir, "self-uuid", filepath.Join(f.selfDir, "node.crt")) + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + slow.respond = func(http.ResponseWriter, *http.Request) bool { + once.Do(func() { close(entered) }) + <-release + return false + } + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: slow.raw}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { f.daemon.authenticateCandidatesLoop(ctx); close(done) }() + t.Cleanup(func() { cancel(); close(release); <-done }) + select { + case <-entered: + case <-time.After(time.Second): + require.FailNow(t, "authentication request to the slow peer did not start") + } + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + f.daemon.refreshTelemetryOnce(context.Background()) + var message Message + require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message)) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, "peer-uuid", sample.HostUUID) + assert.Equal(t, uint32(fakePeerGPUUtilization), sample.GPUUtilizationPct) +} + +func TestDiscovery_TelemetryIncludesSampleAgeAndReusesConnection(t *testing.T) { + const maxSampleAgeMillis = 3500 + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + for range 3 { + f.daemon.refreshTelemetryOnce(context.Background()) + } + decoder := json.NewDecoder(&output) + for range 3 { + var message Message + require.NoError(t, decoder.Decode(&message)) + require.Equal(t, noderec.NotifyNodeTelemetry, message.Method) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, "peer-uuid", sample.HostUUID) + assert.Equal(t, uint32(fakePeerGPUUtilization), sample.GPUUtilizationPct) + assert.True(t, sample.TelemetryValid) + assert.GreaterOrEqual(t, sample.MSSince, int64(fakePeerSampleAgeMillis)) + assert.LessOrEqual(t, sample.MSSince, int64(maxSampleAgeMillis)) + } + assert.Equal(t, int32(1), f.connections.Load()) +} + +// Authenticated remote telemetry must report the maximum across every GPU. +func TestDiscovery_TelemetryReportsHighestGPUUtilization(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.info.GPUs = []GPUInfo{ + {Name: "first GPU", UtilizationPercent: 12}, + {Name: "busiest GPU", UtilizationPercent: 84}, + {Name: "last GPU", UtilizationPercent: 20}, + } + f.discover(t) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + f.daemon.refreshTelemetryOnce(context.Background()) + + message := readTelemetryMessage(t, json.NewDecoder(&output)) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, uint32(84), sample.GPUUtilizationPct) +} + +func TestDiscovery_InvalidTelemetryStaysInvalidAndReportsNoAge(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.info.TelemetryValid, f.info.MSSince = false, 10000 + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + f.daemon.refreshTelemetryOnce(context.Background()) + var message Message + require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message)) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.False(t, sample.TelemetryValid) + assert.Zero(t, sample.MSSince) +} + +func readTelemetryMessage(t *testing.T, decoder *json.Decoder) Message { + t.Helper() + for { + var message Message + require.NoError(t, decoder.Decode(&message)) + if message.Method == noderec.NotifyNodeTelemetry { + return message + } + } +} + +func TestDiscovery_RechecksSavedCertificatesOnReusedConnections(t *testing.T) { + test := func(name string, invalidate func(*testing.T, *discoveryPeerFixture)) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + f.invalidateDuringRead(t, invalidate, func() { f.daemon.refreshTelemetryOnce(context.Background()) }) + assert.Empty(t, output.Bytes(), "invalidated requests must not emit telemetry") + assert.Equal(t, int32(1), f.connections.Load()) + }) + } + test("saved peer certificate removed during response", func(t *testing.T, f *discoveryPeerFixture) { clustertrusttest.RemovePeerPin(t, f.selfDir, "peer-uuid") }) + test("saved peer certificate replaced during response", func(t *testing.T, f *discoveryPeerFixture) { clustertrusttest.WritePeerPin(t, f.selfDir, "peer-uuid") }) +} + +func TestDiscovery_QueuedNodeInfoRequestsGetTheirOwnTimeout(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.daemon.nodeInfoClients.CloseIdle() + f.daemon.nodeInfoClients = clustertrust.NewPeerClientPoolOpts(f.daemon.mesh, clustertrust.PeerClientOptions{ + Timeout: 300 * time.Millisecond, DialTimeout: nodeInfoDialTimeout, MaxConnsPerHost: 1, + }) + t.Cleanup(f.daemon.nodeInfoClients.CloseIdle) + // The requests share one connection, so the second waits for the first. + f.connections.Store(0) + f.respond = func(http.ResponseWriter, *http.Request) bool { time.Sleep(200 * time.Millisecond); return false } + node, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok, "verified node must exist") + var workers sync.WaitGroup + results := make(chan bool, 2) + for range 2 { + workers.Add(1) + go func() { + defer workers.Done() + _, ok := f.daemon.readPeerInfo(context.Background(), node) + results <- ok + }() + } + workers.Wait() + close(results) + for ok := range results { + assert.True(t, ok, "queued healthy request must succeed") + } + assert.Equal(t, int32(1), f.connections.Load()) +} + +func TestDiscovery_TriesNextAddressWhenFirstAddressFails(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.raw.Addresses = []string{"127.0.0.2", "127.0.0.1"} + f.discover(t) + node, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + assert.Equal(t, noderec.IdentityPinned, node.IdentityVerification) +} + +func TestDiscovery_NewSavedCertificateRequiresAuthenticationBeforeTrust(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + clustertrusttest.RemovePeerPin(t, f.selfDir, "peer-uuid") + f.discover(t) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present, "verification requires a saved certificate") + writePinFromCert(t, f.selfDir, "peer-uuid", filepath.Join(f.peerDir, "node.crt")) + f.daemon.reloadTrust() + _, present = f.daemon.dir.get("peer-uuid") + assert.False(t, present, "saving a certificate must not grant trust before authentication") + f.daemon.refreshCandidatesOnce(context.Background()) + node, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.True(t, node.Trusted) +} + +func (f *discoveryPeerFixture) attach(t *testing.T, d *daemon) { + t.Helper() + writePinFromCert(t, d.clusterDir, f.info.HostUUID, filepath.Join(f.peerDir, "node.crt")) + writePinFromCert(t, f.peerDir, "self-uuid", filepath.Join(d.clusterDir, "node.crt")) + f.daemon = d + f.discover(t) +} + +func TestDiscovery_TelemetryRequestsStayWithinConcurrencyLimitAcrossRefreshes(t *testing.T) { + first := newDiscoveryPeerFixture(t, "peer-0") + d := first.daemon + first.discover(t) + fixtures := []*discoveryPeerFixture{first} + for i := 1; i <= telemetryRefreshConcurrency; i++ { + f := newDiscoveryPeerFixture(t, fmt.Sprintf("peer-%d", i)) + f.attach(t, d) + fixtures = append(fixtures, f) + } + entered := make(chan struct{}, len(fixtures)) + release := make(chan struct{}) + var releaseOnce sync.Once + for _, f := range fixtures { + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + entered <- struct{}{} + select { + case <-release: + case <-r.Context().Done(): + } + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { d.runTelemetryLoop(ctx, 5*time.Millisecond); close(done) }() + t.Cleanup(func() { cancel(); releaseOnce.Do(func() { close(release) }); <-done }) + for range telemetryRefreshConcurrency { + select { + case <-entered: + case <-time.After(2 * time.Second): + require.FailNow(t, "telemetry did not start the allowed number of simultaneous requests") + } + } + select { + case <-entered: + require.FailNow(t, "later refreshes exceeded the limit on simultaneous telemetry requests") + case <-time.After(50 * time.Millisecond): + } + releaseOnce.Do(func() { close(release) }) + select { + case <-entered: + case <-time.After(time.Second): + require.FailNow(t, "queued peer did not start when capacity became available") + } +} + +func TestDiscovery_RemovesUnavailablePeerAfterItsDiscoveryExpires(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.server.Close() + delete(f.daemon.lastInfoAt, "peer-uuid") + f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: f.raw}) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) +} + +func TestDiscovery_RemovesExpiredDiscoveryAfterUUIDClaimChanges(t *testing.T) { + test := func(name, uuidClaim string, discover func(*testing.T, *discoveryPeerFixture)) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + discover(t, f) + recordID := nodeDiscoveryRecordID(f.raw) + require.NotNil(t, f.daemon.candidates[recordID]) + f.daemon.publishSelf() + selfUUID := f.daemon.reg.record().HostUUID + self, ok := f.daemon.dir.get(selfUUID) + require.True(t, ok, "local record must exist") + f.server.Close() + delete(f.daemon.lastInfoAt, "peer-uuid") + updatedTXT := make([]string, 0, len(f.raw.TXT)) + for _, field := range f.raw.TXT { + if !strings.HasPrefix(field, "uuid=") { + updatedTXT = append(updatedTXT, field) + } + } + if uuidClaim != "" { + updatedTXT = append(updatedTXT, "uuid="+uuidClaim) + } + f.raw.TXT = updatedTXT + require.Equal(t, recordID, nodeDiscoveryRecordID(f.raw)) + f.daemon.onBrowse(DiscoveryEvent{Type: "updated", Node: f.raw}) + f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: f.raw}) + + assert.Nil(t, f.daemon.candidates[recordID]) + _, present := f.daemon.dir.get(recordID) + assert.False(t, present, "expired unverified discovery must be removed") + _, present = f.daemon.dir.get("peer-uuid") + assert.False(t, present, "expired unavailable peer must be removed") + assert.Empty(t, f.daemon.peerCertificates) + after, present := f.daemon.dir.get(selfUUID) + require.True(t, present) + assert.Equal(t, self, after, "discovery removal must preserve the local record") + }) + } + test("verified peer claims local UUID", "self-uuid", func(t *testing.T, f *discoveryPeerFixture) { + f.discover(t) + }) + test("unverified discovery claims local UUID", "self-uuid", func(t *testing.T, f *discoveryPeerFixture) { + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: f.raw}) + }) + test("verified peer drops UUID", "", func(t *testing.T, f *discoveryPeerFixture) { + f.discover(t) + }) + test("unverified discovery drops UUID", "", func(t *testing.T, f *discoveryPeerFixture) { + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: f.raw}) + }) +} + +func TestDiscovery_RetainedPeerExpiresAfterAvailabilityEvidenceEnds(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + refresh := func() { + var workers sync.WaitGroup + // Advance dispatch time past any failed attempt's retry deadline. + f.daemon.dispatchTelemetry(context.Background(), time.Now().Add(24*time.Hour), make(chan struct{}, telemetryRefreshConcurrency), &workers) + workers.Wait() + } + f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: f.raw}) + assert.Empty(t, f.daemon.candidates) + refresh() + _, present := f.daemon.dir.get("peer-uuid") + require.True(t, present, "reachable peer must survive announcement expiry") + f.server.Close() + f.daemon.lastInfoAt["peer-uuid"] = time.Now().Add(-15 * time.Second) + refresh() + _, present = f.daemon.dir.get("peer-uuid") + require.True(t, present, "temporary node-info failure must retain the peer") + f.daemon.lastInfoAt["peer-uuid"] = time.Now().Add(-2 * time.Minute) + f.daemon.noteActivity("peer-uuid", 0) + refresh() + _, present = f.daemon.dir.get("peer-uuid") + require.True(t, present, "recent inference activity must retain the peer") + f.daemon.lastActivityAt["peer-uuid"] = time.Now().Add(-activityFreshness - time.Second) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + refresh() + _, present = f.daemon.dir.get("peer-uuid") + assert.False(t, present) + assert.Empty(t, f.daemon.peerCertificates) + assert.Empty(t, f.daemon.lastInfoAt) + var message Message + require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message)) + var removed noderec.NodeEvent + require.NoError(t, json.Unmarshal(message.Params, &removed)) + assert.Equal(t, noderec.NotifyNodeRemoved, message.Method) + assert.Equal(t, "peer:peer-uuid", removed.Node.DiscoveryRecordID) + f.daemon.mesh.Refresh() + assert.True(t, f.daemon.mesh.Clustered()) + assert.True(t, f.daemon.mesh.HasPin("peer-uuid")) +} + +func TestDiscovery_FailedRefreshKeepsPeersWithActiveDiscoveries(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.server.Close() + f.daemon.lastInfoAt["peer-uuid"] = time.Now().Add(-2 * time.Minute) + f.daemon.refreshTelemetryOnce(context.Background()) + node, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.True(t, node.Trusted) +} + +func TestDiscovery_RejectsNodeInfoResponsesExceedingSizeLimit(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.respond = func(w http.ResponseWriter, _ *http.Request) bool { + assert.NoError(t, json.NewEncoder(w).Encode(f.info)) + _, err := io.WriteString(w, strings.Repeat(" ", nodeInfoResponseDrainLimit)) + assert.NoError(t, err) + return true + } + f.discover(t) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) +} + +func TestDiscovery_RejectsRedirectsToPublicHTTP(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + var reads atomic.Int32 + public := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reads.Add(1) })) + t.Cleanup(public.Close) + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + http.Redirect(w, r, public.URL, http.StatusTemporaryRedirect) + return true + } + f.discover(t) + assert.Equal(t, int32(0), reads.Load()) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) +} + +type peerProcessPipes struct { + io.Reader + io.Writer +} + +func TestDiscovery_AuthenticatesRealNodeInfoProcess(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.server.Close() + path := filepath.Join(t.TempDir(), "nvpair-node-info") + if runtime.GOOS == "windows" { + path += ".exe" + } + build := exec.Command("go", "build", "-o", path, ".") + build.Dir = "../nvpair-node-info" + output, err := build.CombinedOutput() + require.NoError(t, err, "build node-info: %s", output) + port := closedPort(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + cmd := exec.CommandContext(ctx, path, "--cluster-dir", f.peerDir, "--node-id", "peer-uuid", "--port", strconv.Itoa(port)) + stdin, err := cmd.StdinPipe() + require.NoError(t, err) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + assert.NoError(t, stdin.Close()) + assert.NoError(t, cmd.Wait()) + }) + codec := NewCodec(peerProcessPipes{Reader: stdout, Writer: stdin}) + for { + message, err := codec.Read() + require.NoError(t, err) + if message.Method == noderec.NotifyInfoUpdated { + break + } + } + metadata := noderec.PeerMetadataParams{HostUUID: "peer-uuid", ClusterUUID: "peer-uuid", Peer: noderec.PeerMetadata{ + Name: "owned process metadata", Addresses: []string{"127.0.0.1"}, Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port, noderec.ServiceOllama: fakePeerPort}, + }} + params, err := json.Marshal(metadata) + require.NoError(t, err) + id := json.RawMessage("1") + require.NoError(t, json.NewEncoder(stdin).Encode(Message{JSONRPC: "2.0", ID: &id, Method: noderec.MethodSetPeerMetadata, Params: params})) + for { + message, err := codec.Read() + require.NoError(t, err) + if message.ID == nil { + continue + } + require.Nil(t, message.Error) + break + } + f.raw.TXT = []string{"uuid=peer-uuid", "cluster-uuid=peer-uuid", fmt.Sprintf("ni=%d", port), "ol=1"} + f.discover(t) + node, ok := f.daemon.dir.get("peer-uuid") + require.True(t, ok) + assert.Equal(t, noderec.IdentityPinned, node.IdentityVerification) + assert.Equal(t, metadata.Peer.Name, node.Name) + assert.Equal(t, fakePeerPort, node.Services[noderec.ServiceOllama].Port) + public := &http.Client{Timeout: time.Second} + t.Cleanup(public.CloseIdleConnections) + resp, err := public.Get("http://127.0.0.1:" + strconv.Itoa(port) + "/v1/node-info") + require.NoError(t, err) + defer resp.Body.Close() + var info NodeInfoResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&info)) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "peer-uuid", info.HostUUID) + assert.Nil(t, info.Peer) +} + +func TestDiscovery_UnverifiedAddressesCannotInfluenceLocalAddressSelection(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.daemon.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "rogue", Addresses: []string{"192.0.2.30"}, TXT: []string{"uuid=peer-uuid", "ni=14318"}}}) + assert.Equal(t, []string{"127.0.0.1", "127.0.0.1"}, f.daemon.peerAddresses()) +} + +func TestDiscovery_FailedAnnouncementUpdateStillAllowsRemovalOfUnavailablePeer(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.server.Close() + delete(f.daemon.lastInfoAt, "peer-uuid") + updated := f.raw + updated.TXT = append(append([]string(nil), f.raw.TXT...), "er=14319") + f.daemon.onBrowse(DiscoveryEvent{Type: "updated", Node: updated}) + f.daemon.refreshCandidatesOnce(context.Background()) + f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: updated}) + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) +} + +func TestRefreshNodeModels_AcceptsPinnedPeersWithMultipleAddresses(t *testing.T) { + const peerUUID = "peer-uuid" + f := newDiscoveryPeerFixture(t, peerUUID) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + f.info.Peer.Addresses = []string{"127.0.0.1", "127.0.0.2"} + type modelResponse struct { + Models []string `json:"models"` + } + response := modelResponse{Models: []string{"reported-model"}} + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(response)) + return true + } + f.discover(t) + + assert.True(t, f.daemon.refreshNodeModels(peerUUID, "127.0.0.1", port, peerUUID)) + + node, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + assert.Equal(t, response.Models, node.Models) +} + +func TestDiscovery_DiscardsModelsAfterSecondaryAddressChanges(t *testing.T) { + const peerUUID = "peer-uuid" + const primaryAddress = "127.0.0.2" + + test := func(name, secondaryAddress string, wantAccepted bool) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, peerUUID) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + // Only the secondary address reaches this peer's server. + f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.1"} + type modelResponse struct { + Models []string `json:"models"` + } + response := modelResponse{Models: []string{"reported-model"}} + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(response)) + return true + } + f.discover(t) + original, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + client, ok := f.daemon.modelsClients.Client(peerUUID) + require.True(t, ok) + closed, release := make(chan struct{}), make(chan struct{}) + client.Transport = heldModelResponseTransport{RoundTripper: client.Transport, closed: closed, release: release} + changed := make(chan bool, 1) + done := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) + go func() { + defer close(done) + changed <- f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID) + }() + select { + case <-closed: + case <-time.After(8 * time.Second): + require.FailNow(t, "model response did not reach the point before directory update") + } + + // Refresh authoritative inventory while the old model result waits to + // be applied. The primary address, port, and pin remain unchanged. + f.info.Peer.Addresses = []string{primaryAddress, secondaryAddress} + require.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), peerUUID, original.CandidateIPs(), original.Services[noderec.ServiceNodeInfo].Port)) + before, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + require.Equal(t, []string{primaryAddress, secondaryAddress}, before.CandidateIPs()) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + releaseOnce.Do(func() { close(release) }) + <-done + + assert.Equal(t, wantAccepted, <-changed) + after, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + if wantAccepted { + assert.Equal(t, response.Models, after.Models) + return + } + assert.Equal(t, before, after) + assert.Empty(t, output.Bytes()) + }) + } + + test("unchanged addresses accept the model response", "127.0.0.1", true) + test("removed secondary address discards the model response", "127.0.0.3", false) +} + +func TestDiscovery_DiscardsQueuedModelsForChangedAddresses(t *testing.T) { + const peerUUID = "peer-uuid" + const primaryAddress = "127.0.0.2" + f := newDiscoveryPeerFixture(t, peerUUID) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.1"} + type modelResponse struct { + Models []string `json:"models"` + } + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(modelResponse{Models: []string{"obsolete-model"}})) + return true + } + f.discover(t) + original, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + + // The model sweep queued the old addresses before receiving new inventory. + f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.3"} + require.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), peerUUID, original.CandidateIPs(), original.Services[noderec.ServiceNodeInfo].Port)) + before, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + assert.False(t, f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID)) + + after, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + assert.Equal(t, before, after) + assert.Empty(t, output.Bytes()) } diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index 613e0fe6a..0afb17045 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -17,25 +17,9 @@ import ( "nvpair-shared/noderec" ) -// toDirectoryNode converts a browsed _nvpair-node instance into a directory entry -// — identity + advertised services — without enrichment or liveness (those -// decorate it later in the daemon). The directory is keyed by hostUuid (the -// stable identity invite/attribution correlate on); name is the mDNS instance -// (hostname). trusted is supplied by the caller from the cluster pin set. -// -// Addresses are carried through whole rather than reduced to one. The node ranked -// its own addresses from evidence no observer has — which of its interfaces the -// kernel routes off-link, which carry live peers, which ones peers have connected -// to — so IPs keeps that order and IP is simply its head. Collapsing here is what -// left every consumer with a single address and no recourse when it turned out to -// be a link only one other machine could reach. -// -// It returns ok=false for a record without a uuid= TXT: a conformant -// _nvpair-node record always carries one (the daemon stamps it from -// nodeid.Resolve), so a record without it can't be keyed by a stable identity. -// We skip it (and warn) rather than key it by the mutable, non-unique hostname — -// node identity is the UUID everywhere, with no hostname fallback. -func toDirectoryNode(raw RawNode, trusted bool) (noderec.DirectoryNode, bool) { +// toDirectoryNode projects announcement claims for display and authentication. +// A claimed UUID or principal never grants trust. +func toDirectoryNode(raw RawNode) (noderec.DirectoryNode, bool) { rec := noderec.ParseTXT(raw.TXT) if rec.HostUUID == "" { slog.Warn("skipping _nvpair-node record without uuid=; cannot key it by a stable identity", @@ -61,10 +45,9 @@ func toDirectoryNode(raw RawNode, trusted bool) (noderec.DirectoryNode, bool) { IP: ip, IPs: ips, ClusterUUID: rec.ClusterUUID, - Trusted: trusted, Services: svcs, - // Models is enriched separately from engine-manager's em /v1/models - // (daemon.enrichModelsAt), not carried in the record's TXT. + // Models are absent from announcements. Only the verified peer record + // receives them through the authenticated engine-manager refresh. LastSeen: time.Now().Unix(), }, true } @@ -80,9 +63,10 @@ func nodeDiscoveryRecordID(raw RawNode) string { return fmt.Sprintf("observation:%x", sha256.Sum256([]byte(fmt.Sprintf("%q", fields)))) } -// directory stores this node and discovered peers by hostUuid. Browse events, -// IPC handlers and background refreshes access it concurrently, so reads and -// mutations share mu. +// directory keeps canonical hosts by UUID and unverified discoveries by their +// scanner-assigned record ID. A network claim cannot select a canonical key. +// The mutex covers whole changes, such as removing a discovery record and storing +// its verified peer. sync.Map alone only protects individual map operations. type directory struct { mu sync.RWMutex nodes map[string]noderec.DirectoryNode @@ -96,118 +80,40 @@ func newDirectory() *directory { func (d *directory) upsert(n noderec.DirectoryNode) (isNew bool) { d.mu.Lock() defer d.mu.Unlock() - _, existed := d.nodes[n.HostUUID] - d.nodes[n.HostUUID] = n + _, existed := d.nodes[directoryKey(n)] + d.nodes[directoryKey(n)] = n return !existed } -// supersedeMinAge is how much older a record must be than the one arriving at -// its address before it is worth asking whether it has been replaced. -// -// Note what it is NOT: evidence. LastSeen is not a liveness clock — the browser -// reports a node only when its record CHANGES (shared/discovery, reconcile), so -// a healthy peer with a stable advertisement produces no events and its LastSeen -// freezes at first discovery exactly like a ghost's does. A frozen timestamp -// says nothing about whether the node is alive, and anything willing to wait out -// the window satisfies it. Nothing may be deleted on the strength of it; it -// earns its place only as a cheap filter that keeps the identity probe in -// daemon.supersedingUpsert off the common path. -const supersedeMinAge int64 = 300 // seconds, matching DirectoryNode.LastSeen - -// supersedeCandidates lists the records a newly browsed node MIGHT have replaced, -// for the caller to confirm or reject with proof. It decides nothing and deletes -// nothing. -// -// This is the peer-side counterpart to dropSelf: when a node's appdata is wiped -// it mints a fresh hostUuid, so the fleet sees the same machine arrive as a -// stranger while the pre-wipe record lingers under the old key. dropSelf handles -// that re-stamp for the local node; nothing handled it for peers, so a reset -// node showed up twice everywhere downstream. -// -// A record is nominated — and no more than nominated — when it shares the -// arriving node's Name, carries a different hostUuid, and is at least -// supersedeMinAge older: -// -// - Same Name. A wipe clears identity but not the hostname. -// - Older than supersedeMinAge (a filter, not evidence — see above). -// -// A shared address is deliberately NOT part of this. It reads as "one machine" -// only if an address belongs to one machine at a time, and it does not: a -// direct-connect link between two machines uses the same small subnet on every -// such pair, so identically-provisioned hosts genuinely advertise the same -// address on it, and matching there nominated live peers as each other's ghosts. -// Nor is it needed — a wiped machine keeps its hostname whether or not it comes -// back on the same address. -// -// Both remaining signals come from an unauthenticated mDNS record that anything -// on the LAN can claim, so together they describe the SHAPE of a wipe and equally -// the shape of a spoof — and, now that the address is gone, also the shape of two -// hosts that merely share a hostname. daemon.supersedingUpsert must confirm a -// candidate is really gone before deleting it. -// -// The arriving record still needs an address of its own, since that is where the -// confirming read is made — not because matching it against anything would mean -// something. -// -// selfUUID is never nominated, and never nominates: the local record is -// registry-driven rather than browse-driven, and its LastSeen advances only when -// this node republishes, so it must not be displaceable by a peer that happens -// to claim its name. -// -// Candidates are returned sorted by hostUuid so the resulting removals are -// emitted in a deterministic order rather than map-iteration order. -func (d *directory) supersedeCandidates(n noderec.DirectoryNode, selfUUID string) []noderec.DirectoryNode { - if n.IP == "" || n.Name == "" || n.HostUUID == selfUUID { - return nil - } - d.mu.RLock() - defer d.mu.RUnlock() - var candidates []noderec.DirectoryNode - for uuid, other := range d.nodes { - if uuid == n.HostUUID || uuid == selfUUID { - continue - } - if other.Name != n.Name { - continue - } - if n.LastSeen-other.LastSeen < supersedeMinAge { - continue - } - candidates = append(candidates, other) +// directoryKey keeps unverified claims outside the canonical host namespace. +func directoryKey(n noderec.DirectoryNode) string { + if n.IdentityVerification == noderec.IdentityUnverified { + return n.DiscoveryRecordID } - sort.Slice(candidates, func(i, j int) bool { return candidates[i].HostUUID < candidates[j].HostUUID }) - return candidates + return n.HostUUID } -// upsertEvicting stores a node and deletes the records the caller has PROVEN it -// replaced, returning the entries actually removed so the caller can emit -// removals and drop their caches. -// -// The proof is a network read, so it is gathered without this lock held and each -// eviction is re-validated here against the entry the caller actually judged. A -// record that was removed, re-addressed, renamed, or refreshed while the probe -// was in flight is left alone: the state the verdict was formed against is gone, -// and it must be judged again from the new one. -func (d *directory) upsertEvicting(n noderec.DirectoryNode, proven []noderec.DirectoryNode) (evicted []noderec.DirectoryNode) { +// promote atomically replaces the authenticated discovery with its peer record. +// Inventory refreshes preserve model fields written concurrently by their owner. +func (d *directory) promote(candidateID string, n noderec.DirectoryNode) (removed noderec.DirectoryNode, discovered, changed bool) { d.mu.Lock() defer d.mu.Unlock() - for _, old := range proven { - if old.HostUUID == n.HostUUID { - continue - } - current, present := d.nodes[old.HostUUID] - if !present || current.IP != old.IP || current.Name != old.Name || current.LastSeen != old.LastSeen { - continue - } - delete(d.nodes, old.HostUUID) - evicted = append(evicted, current) + removed = d.nodes[candidateID] + delete(d.nodes, candidateID) + old, present := d.nodes[n.HostUUID] + if present { + n.Models, n.ModelsByEngine, n.LoadedByEngine = old.Models, old.ModelsByEngine, old.LoadedByEngine + n.LastSeen = old.LastSeen + } + changed = !present || !old.Equals(n) + if changed { + n.LastSeen = time.Now().Unix() } - sort.Slice(evicted, func(i, j int) bool { return evicted[i].HostUUID < evicted[j].HostUUID }) d.nodes[n.HostUUID] = n - return evicted + return removed, !present, changed } -// remove drops a node by hostUuid and reports whether it existed. +// remove drops a canonical host or a discovery by its storage key. func (d *directory) remove(hostUUID string) (existed bool) { d.mu.Lock() defer d.mu.Unlock() @@ -216,7 +122,7 @@ func (d *directory) remove(hostUUID string) (existed bool) { return existed } -// get returns a node by hostUuid. +// get returns a canonical host by UUID or an unverified discovery by record ID. func (d *directory) get(hostUUID string) (noderec.DirectoryNode, bool) { d.mu.RLock() defer d.mu.RUnlock() @@ -237,7 +143,12 @@ func (d *directory) snapshot(filter noderec.ServiceKey) []noderec.DirectoryNode out = append(out, n) } d.mu.RUnlock() - sort.Slice(out, func(i, j int) bool { return out[i].HostUUID < out[j].HostUUID }) + sort.Slice(out, func(i, j int) bool { + if out[i].HostUUID == out[j].HostUUID { + return out[i].DiscoveryRecordID < out[j].DiscoveryRecordID + } + return out[i].HostUUID < out[j].HostUUID + }) return out } @@ -278,17 +189,16 @@ func equalOptional[T comparable](a, b *T) bool { // applyModels atomically reconciles one node's model inventory from a completed // /v1/models fetch, guarded against a node that was removed or re-addressed -// while the fetch was in flight. It is the periodic refresh loop's write path -// (daemon.refreshNodeModels); the event-driven enrichment writes the whole node -// via upsert instead. +// while the fetch was in flight. Both the post-authentication model sweep and +// periodic refresh use this write path through refreshNodeModelsCandidates. // // It confirms the node still exists AND still advertises the same em endpoint // (ip + port) the fetch targeted, so an in-flight result can never resurrect a // removed node or overwrite one that has since been re-addressed. It then // compares the fetched flat, per-engine, and loaded inventories // order-insensitively with the stored entry and, only when any changed, updates -// just the model fields in place — preserving every other field (identity, -// services, node-info, trust) the browse path owns. +// just the model fields in place, preserving identity, services, hardware, and +// trust from local registry publications or authenticated peer inventory. // // Returns the (possibly updated) node, whether the inventory changed, and // whether the guarded apply was valid. ok == false means the result is stale @@ -314,51 +224,6 @@ func (d *directory) applyModels(hostUUID, ip string, emPort int, models []string return n, true, true } -// applyClusterIdentity sets one node's cluster principal and the trust annotation -// derived from it, reporting whether either changed. It is the write path for both -// membership reconciles — the pin-set pass (daemon.reconcileTrust) and the -// node-info pass (daemon.refreshClusterIdentityOnce) — and mirrors applyModels: it -// updates just those two fields in place, preserving everything the browse and -// enrichment paths own, and reports no change for an entry that already agrees so -// the caller stays silent in the steady state. -// -// The two fields move together because trust is derived from the principal: a peer -// that leaves its cluster is no longer pinned under it, so leaving one stale while -// updating the other would describe a node that cannot exist. For the same reason -// trust is computed here, from the principal this call settles on, rather than -// supplied by the caller: a caller computing it from a snapshot would race the -// other reconcile and could write back a principal that had already moved. -// -// A nil clusterUUID means "keep whatever is stored" — the pin-set pass re-derives -// trust without claiming to know the principal, since only the peer's own record -// or report can say what it is. -// -// A node that has since been evicted reports no change — there is nothing to -// annotate, and a later rediscovery evaluates both from scratch. -func (d *directory) applyClusterIdentity(hostUUID string, clusterUUID *string, trustFor func(clusterUUID string) bool) (node noderec.DirectoryNode, changed bool) { - d.mu.Lock() - defer d.mu.Unlock() - n, present := d.nodes[hostUUID] - if !present { - return n, false - } - settled := n.ClusterUUID - if clusterUUID != nil { - settled = *clusterUUID - } - trusted := false - if settled != "" { - trusted = trustFor(settled) - } - if n.ClusterUUID == settled && n.Trusted == trusted { - return n, false - } - n.ClusterUUID = settled - n.Trusted = trusted - d.nodes[hostUUID] = n - return n, true -} - // sameByEngine reports whether two per-engine model maps are semantically equal: // the same set of engine keys, each mapping to the same set of model names // (order-insensitive). A nil and an empty map compare equal (both mean "no diff --git a/services/nvpair-node-scanner/directory_test.go b/services/nvpair-node-scanner/directory_test.go index 777e92508..745a09c31 100644 --- a/services/nvpair-node-scanner/directory_test.go +++ b/services/nvpair-node-scanner/directory_test.go @@ -4,6 +4,7 @@ package main import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -35,22 +36,54 @@ func TestApplyLocalInfo_UpdatesOnlyHardware(t *testing.T) { assert.Equal(t, want, got) } -func TestToDirectoryNode_ProjectsIdentityServicesAndTrust(t *testing.T) { +func TestDirectory_IdenticalPeerRefreshPreservesModelsAndTimestamp(t *testing.T) { + dir := newDirectory() + old := noderec.DirectoryNode{ + HostUUID: "peer-uuid", DiscoveryRecordID: "peer:peer-uuid", + IdentityVerification: noderec.IdentityPinned, ClusterUUID: "peer-uuid", Trusted: true, + Name: "peer", IP: "192.0.2.20", IPs: []string{"192.0.2.20"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: 14318}}, + GPUs: []GPUInfo{{Name: "GPU", UtilizationPercent: 61}}, + CPU: &CPUInfo{Name: "CPU"}, + Memory: &MemoryInfo{TotalBytes: 1024}, + Models: []string{"current-model"}, + ModelsByEngine: map[string][]string{"ollama": {"current-model"}}, + LoadedByEngine: map[string][]string{"ollama": {"current-model"}}, + LastSeen: 1, + } + dir.upsert(old) + // A new response has equal values in separately allocated slices, maps, + // and hardware pointers. It supplies neither models nor a change timestamp. + body, err := json.Marshal(old) + require.NoError(t, err) + var refreshed noderec.DirectoryNode + require.NoError(t, json.Unmarshal(body, &refreshed)) + refreshed.Models, refreshed.ModelsByEngine, refreshed.LoadedByEngine = nil, nil, nil + refreshed.LastSeen = 0 + _, discovered, changed := dir.promote("", refreshed) + assert.False(t, discovered) + assert.False(t, changed) + got, present := dir.get(old.HostUUID) + require.True(t, present) + assert.Equal(t, old, got) +} + +func TestToDirectoryNode_ProjectsClaimsWithoutGrantingTrust(t *testing.T) { raw := RawNode{ ID: "hostA", Host: "hostA.local.", Addresses: []string{"10.221.0.9", "192.168.1.10"}, TXT: []string{"v=1", "uuid=host-uuid", "cluster-uuid=clu", "ip=192.168.1.10", "ni=14318", "ol=11434"}, } - n, ok := toDirectoryNode(raw, true) + n, ok := toDirectoryNode(raw) require.True(t, ok, "a record with uuid= should project") assert.Equal(t, "host-uuid", n.HostUUID) assert.Equal(t, "hostA", n.Name) assert.Equal(t, "192.168.1.10", n.IP) - assert.True(t, n.Trusted, "Trusted should reflect the caller-supplied flag") - assert.True(t, n.HasService(noderec.ServiceNodeInfo), "services wrong") - assert.True(t, n.HasService(noderec.ServiceOllama), "services wrong") - assert.Equal(t, 11434, n.Services[noderec.ServiceOllama].Port, "ol port") + assert.False(t, n.Trusted) + assert.True(t, n.HasService(noderec.ServiceNodeInfo)) + assert.True(t, n.HasService(noderec.ServiceOllama)) + assert.Equal(t, 11434, n.Services[noderec.ServiceOllama].Port) } // TestToDirectoryNodeIgnoresSRVPort asserts the locked-decision contract: the @@ -66,10 +99,10 @@ func TestToDirectoryNode_IgnoresSRVPort(t *testing.T) { Addresses: []string{"192.168.1.30"}, TXT: []string{"v=1", "uuid=host-c", "ip=192.168.1.30", "ni=14318", "ol=11434"}, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) require.True(t, ok, "a record with uuid= should project") - assert.Equal(t, 14318, n.Services[noderec.ServiceNodeInfo].Port, "ni port") - assert.Equal(t, 11434, n.Services[noderec.ServiceOllama].Port, "ol port") + assert.Equal(t, 14318, n.Services[noderec.ServiceNodeInfo].Port) + assert.Equal(t, 11434, n.Services[noderec.ServiceOllama].Port) for svc, st := range n.Services { assert.NotEqual(t, 65000, st.Port, "the non-authoritative SRV port leaked into service (%v)", svc) } @@ -84,8 +117,8 @@ func TestToDirectoryNode_SkipsWithoutUUID(t *testing.T) { Addresses: []string{"10.221.0.9", "192.168.1.20"}, TXT: []string{"v=1", "er=14319"}, } - _, ok := toDirectoryNode(raw, false) - assert.False(t, ok, "a record without uuid= must be skipped, not projected") + _, ok := toDirectoryNode(raw) + assert.False(t, ok) } // TestToDirectoryNodeIPFallback: with a uuid= but no ip=, every advertised address @@ -101,11 +134,11 @@ func TestToDirectoryNode_UsesRecordIPWhenAddressesAreAbsent(t *testing.T) { Addresses: []string{"10.221.0.9", "192.168.1.20"}, TXT: []string{"v=1", "uuid=host-b", "er=14319"}, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) require.True(t, ok, "a record with uuid= should project") - assert.Equal(t, "10.221.0.9", n.IP, "IP fallback") + assert.Equal(t, "10.221.0.9", n.IP) assert.Equal(t, []string{"10.221.0.9", "192.168.1.20"}, n.IPs) - assert.False(t, n.Clustered(), "no cluster-uuid should mean not clustered") + assert.False(t, n.Clustered()) } // TestToDirectoryNodePreservesPublishedOrder: a node's own ips= order survives @@ -121,9 +154,9 @@ func TestToDirectoryNode_PreservesPublishedOrder(t *testing.T) { "ips=10.172.54.70,192.168.240.2", "ni=14318", }, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) require.True(t, ok, "a record with uuid= should project") - assert.Equal(t, "10.172.54.70", n.IP, "canonical") + assert.Equal(t, "10.172.54.70", n.IP) assert.Equal(t, []string{"10.172.54.70", "192.168.240.2"}, n.IPs) } @@ -135,9 +168,9 @@ func TestToDirectoryNode_UnionsUnpublishedAddresses(t *testing.T) { Addresses: []string{"10.0.9.9"}, TXT: []string{"v=1", "uuid=host-c", "ip=10.0.0.5", "ips=10.0.0.5"}, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) require.True(t, ok, "a record with uuid= should project") - assert.Equal(t, []string{"10.0.0.5", "10.0.9.9"}, n.IPs, "candidates") + assert.Equal(t, []string{"10.0.0.5", "10.0.9.9"}, n.IPs) } func TestDirectory_UpdatesRemovesAndSnapshotsNodes(t *testing.T) { @@ -177,10 +210,10 @@ func observationFixture() RawNode { func TestToDirectoryNode_AddressOrderAndClaimChangesPreserveDiscoveryRecordID(t *testing.T) { test := func(name string, candidate RawNode) { t.Run(name, func(t *testing.T) { - original, ok := toDirectoryNode(observationFixture(), false) - require.True(t, ok, "original observation must be projected") - next, ok := toDirectoryNode(candidate, false) - require.True(t, ok, "candidate observation must be projected") + original, ok := toDirectoryNode(observationFixture()) + require.True(t, ok, "original observation was not projected") + next, ok := toDirectoryNode(candidate) + require.True(t, ok, "candidate observation was not projected") require.NotEmpty(t, original.DiscoveryRecordID) assert.Equal(t, original.DiscoveryRecordID, next.DiscoveryRecordID) }) @@ -198,10 +231,10 @@ func TestToDirectoryNode_AddressOrderAndClaimChangesPreserveDiscoveryRecordID(t func TestToDirectoryNode_DifferencesInDiscoveryFieldsYieldDifferentDiscoveryRecordIDs(t *testing.T) { test := func(name string, candidate RawNode) { t.Run(name, func(t *testing.T) { - original, ok := toDirectoryNode(observationFixture(), false) - require.True(t, ok, "original observation must be projected") - next, ok := toDirectoryNode(candidate, false) - require.True(t, ok, "candidate observation must be projected") + original, ok := toDirectoryNode(observationFixture()) + require.True(t, ok, "original observation was not projected") + next, ok := toDirectoryNode(candidate) + require.True(t, ok, "candidate observation was not projected") require.NotEmpty(t, original.DiscoveryRecordID) require.NotEmpty(t, next.DiscoveryRecordID) assert.NotEqual(t, original.DiscoveryRecordID, next.DiscoveryRecordID) @@ -225,8 +258,8 @@ func TestToDirectoryNode_DifferencesInDiscoveryFieldsYieldDifferentDiscoveryReco test("different node-info port", port) } -func TestToDirectoryNode_TrustedFlagDoesNotVerifyIdentity(t *testing.T) { - node, ok := toDirectoryNode(observationFixture(), true) - require.True(t, ok, "observation must be projected") +func TestToDirectoryNode_ClaimsNeverVerifyIdentity(t *testing.T) { + node, ok := toDirectoryNode(observationFixture()) + require.True(t, ok, "observation was not projected") assert.Equal(t, noderec.IdentityUnverified, node.IdentityVerification) } diff --git a/services/nvpair-node-scanner/discovery.go b/services/nvpair-node-scanner/discovery.go index 384994b38..97f6295c1 100644 --- a/services/nvpair-node-scanner/discovery.go +++ b/services/nvpair-node-scanner/discovery.go @@ -3,12 +3,9 @@ package main -// The mDNS browser now lives in the shared nvpair-shared/discovery package (the -// mDNS dedup) — nvpair-node-scanner was the canonical source it was lifted from, so -// behavior is identical (5s scans, the shared miss threshold, order-insensitive -// compare, uuid= collision warning, evict at threshold). These aliases keep -// scanner.go and main.go on the local names. The order-insensitivity and -// UUIDFromTXT tests moved to the shared package. +// These aliases expose the shared mDNS browser to the scanner. The daemon keys +// discoveries by discoveryRecordId and supplies authenticated peer availability +// checks. Scan timing and missed-scan limits use the shared defaults. import "nvpair-shared/discovery" @@ -21,18 +18,16 @@ type ( // NewDiscovery builds the node browser with the shared defaults. var NewDiscovery = discovery.New -// WithLivenessProbe re-exports the shared option so the daemon can wire its -// TCP-probe-before-evict anti-flap guard — the guard the proxies -// used to run themselves before discovery was consolidated onto the daemon. +// WithLivenessProbe lets the daemon retain a discovery using its verified peer's +// authenticated inventory or approved inference activity. var WithLivenessProbe = discovery.WithLivenessProbe -// WithKeyFunc re-exports the shared option so the daemon keys its node map by -// uuid= rather than the mDNS instance name (matching cluster-manager). Two hosts -// that share a hostname but hold distinct UUIDs must not collapse into one entry. +// WithKeyFunc lets the daemon key discoveries by discoveryRecordId, independently +// of their claimed UUIDs. Distinct connection candidates remain separate. var WithKeyFunc = discovery.WithKeyFunc -// UUIDFromTXT re-exports the shared uuid= extractor, used as the browser's key -// function (see WithKeyFunc). +// UUIDFromTXT extracts the claimed host UUID, including for filtering this host's +// own announcements. It does not determine the discovery record ID. var UUIDFromTXT = discovery.UUIDFromTXT // sameStringSet re-exports the shared order-insensitive multiset compare so the diff --git a/services/nvpair-node-scanner/enrich_hosts.go b/services/nvpair-node-scanner/enrich_hosts.go index 7997a192f..bb218eae9 100644 --- a/services/nvpair-node-scanner/enrich_hosts.go +++ b/services/nvpair-node-scanner/enrich_hosts.go @@ -4,18 +4,21 @@ package main import ( + "crypto/x509" "log/slog" + "slices" "sync" "nvpair-shared/noderec" ) -// modelInventory is the three-part answer a node's engine manager gives, carried -// as one value so the concurrent ask can return it. +// modelInventory carries engine-manager's model lists and, for remote responses, +// the certificate used to authenticate the server. type modelInventory struct { models []string byEngine map[string][]string loadedByEngine map[string][]string + certificate *x509.Certificate } // hostKey identifies one node's endpoint for one service. Node-info and the engine @@ -72,8 +75,10 @@ func (m *hostMemory) forget(key hostKey) { // different machine, which is not an answer for this one. func askRemembered[T any](mem *hostMemory, key hostKey, hosts []string, ask func(host string) (T, bool)) (string, T, bool) { if remembered := mem.get(key); remembered != "" { - if value, ok := ask(remembered); ok { - return remembered, value, true + if slices.Contains(hosts, remembered) { + if value, ok := ask(remembered); ok { + return remembered, value, true + } } mem.forget(key) slog.Debug("enrichment: the remembered address stopped answering; trying the rest of the list", diff --git a/services/nvpair-node-scanner/enrich_hosts_test.go b/services/nvpair-node-scanner/enrich_hosts_test.go index 6132bb87e..5cf849c52 100644 --- a/services/nvpair-node-scanner/enrich_hosts_test.go +++ b/services/nvpair-node-scanner/enrich_hosts_test.go @@ -82,6 +82,15 @@ func TestAskRemembered_AsksOnlyTheAddressThatAnswered(t *testing.T) { } } +func TestAskRemembered_SkipsWithdrawnAddresses(t *testing.T) { + var memory hostMemory + key := hostKey{hostUUID: "peer", service: noderec.ServiceNodeInfo} + memory.set(key, "old-address") + host, _, ok := askRemembered(&memory, key, []string{"new-address"}, func(host string) (string, bool) { return host, true }) + require.True(t, ok) + assert.Equal(t, "new-address", host) +} + // TestAskRememberedKeepsAWorkingAddressWhenABetterRankedOneComesUp: nothing is // wrong with the address in use, and switching changes the address this node is // enriched and reported at to learn nothing. diff --git a/services/nvpair-node-scanner/ghost_node_test.go b/services/nvpair-node-scanner/ghost_node_test.go deleted file mode 100644 index e702b76d6..000000000 --- a/services/nvpair-node-scanner/ghost_node_test.go +++ /dev/null @@ -1,422 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "fmt" - "net" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "nvpair-shared/noderec" -) - -// A node whose appdata is wiped mints a fresh hostUuid but keeps its hostname -// and re-binds the same fixed service ports. These tests cover the two defences -// against the duplicate ("ghost") record that leaves behind: the liveness probe -// must notice the machine is no longer the node the record describes, and a -// record arriving under that hostname must collapse the one it replaced without -// waiting for the miss threshold. -// -// Both defences delete only on the same proof — a node-info answer naming a -// different host. The directory's hostname signal comes from an unauthenticated -// mDNS record, so it nominates suspects and nothing more; the address is not a -// signal at all, since machines on a direct-connect link legitimately share one. - -// nodeInfoServer starts a /v1/node-info endpoint reporting hostUUID and returns -// its host and port. An empty hostUUID omits the field, mimicking a peer that -// predates node-info reporting identity. -func nodeInfoServer(t *testing.T, hostUUID string) (host string, port int) { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: hostUUID})) - })) - t.Cleanup(srv.Close) - return splitHostPort(t, strings.TrimPrefix(srv.URL, "http://")) -} - -// openPort leaves a bare TCP listener open and returns its port. It stands in -// for a service that accepts connections but proves nothing about identity. -func openPort(t *testing.T) int { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err, "listen") - t.Cleanup(func() { assert.NoError(t, ln.Close()) }) - _, port := splitHostPort(t, ln.Addr().String()) - return port -} - -// closedPort returns a port nothing is listening on, by binding and releasing -// it. Using a bound-then-released port rather than a low well-known one keeps -// the dial a prompt refusal on every platform. -func closedPort(t *testing.T) int { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err, "listen") - _, port := splitHostPort(t, ln.Addr().String()) - require.NoError(t, ln.Close(), "close") - return port -} - -func splitHostPort(t *testing.T, addr string) (string, int) { - t.Helper() - host, portStr, err := net.SplitHostPort(addr) - require.NoError(t, err, "split (%v, %v)", addr, err) - port, err := strconv.Atoi(portStr) - require.NoError(t, err, "port (%v, %v)", portStr, err) - return host, port -} - -// probeDaemon is the minimum daemon the liveness probe needs. -func probeDaemon() *daemon { - return &daemon{http: &http.Client{Timeout: nodeInfoFetchTimeout}} -} - -func rawNode(txt ...string) RawNode { - return RawNode{ID: "wiped-host", Addresses: []string{"127.0.0.1"}, TXT: txt} -} - -// TestReachableRejectsSupersededIdentity is the core of the fix. The ghost's -// cached address and ports are live — they belong to the machine's NEW -// incarnation — so a TCP-only probe reports the stale record alive forever. The -// probe must ask WHO answers, not just WHETHER anything answers. -func TestReachable_RejectsSupersededIdentity(t *testing.T) { - host, port := nodeInfoServer(t, "new-uuid-after-wipe") - d := probeDaemon() - - n := rawNode("v=1", "uuid=old-uuid-before-wipe", "ip="+host, fmt.Sprintf("ni=%d", port)) - require.False(t, d.reachable(n), "a record whose address now answers with a DIFFERENT hostUuid must not be kept alive") -} - -// TestReachableAcceptsMatchingIdentity guards the other direction: a peer that -// merely dropped off multicast must survive the probe, or a transient mDNS gap -// would evict healthy nodes. -func TestReachable_AcceptsMatchingIdentity(t *testing.T) { - host, port := nodeInfoServer(t, "steady-uuid") - d := probeDaemon() - - n := rawNode("v=1", "uuid=steady-uuid", "ip="+host, fmt.Sprintf("ni=%d", port)) - require.True(t, d.reachable(n), "a record whose address confirms the same hostUuid must be kept alive") -} - -// TestReachableFallsBackWithoutIdentity covers every way identity can be -// unavailable. None of them may change today's behaviour: an open ni/em port -// still means alive, so no node that used to survive a probe starts evicting. -// Inference ports (ol/lm) are never used as the TCP fallback. -func TestReachable_FallsBackWithoutIdentity(t *testing.T) { - tcpPort := openPort(t) - niHost, niPort := nodeInfoServer(t, "") // answers, but reports no identity - - test := func(name string, txt ...string) { - t.Run(name, func(t *testing.T) { - require.True(t, probeDaemon().reachable(rawNode(txt...)), "with identity unavailable the probe must fall back to the TCP sweep") - }) - } - deadPort := closedPort(t) - test("no ni advertised", "v=1", "uuid=some-uuid", "ip=127.0.0.1", fmt.Sprintf("em=%d", tcpPort)) - test("node-info not answering", "v=1", "uuid=some-uuid", "ip=127.0.0.1", fmt.Sprintf("ni=%d", deadPort), fmt.Sprintf("em=%d", tcpPort)) - test("node-info reports no hostUuid", "v=1", "uuid=some-uuid", "ip="+niHost, fmt.Sprintf("ni=%d", niPort), fmt.Sprintf("em=%d", tcpPort)) - test("record carries no uuid to compare", "v=1", "ip=127.0.0.1", fmt.Sprintf("ni=%d", niPort), fmt.Sprintf("em=%d", tcpPort)) -} - -// TestReachableEvictsWhenNothingAnswers keeps the base case honest: a node that -// is genuinely gone still evicts. -func TestReachable_EvictsWhenNothingAnswers(t *testing.T) { - d := probeDaemon() - dead := rawNode("v=1", "uuid=gone", "ip=127.0.0.1", - fmt.Sprintf("ni=%d", closedPort(t)), fmt.Sprintf("ol=%d", closedPort(t))) - require.False(t, d.reachable(dead), "a node answering on nothing must not be kept alive") - require.False(t, d.reachable(rawNode("v=1", "uuid=identity-only", "ip=127.0.0.1")), "a node advertising no services has nothing to probe and must evict") -} - -// countingTransport records how many HTTP requests the probe actually issues. -type countingTransport struct { - n int -} - -func (c *countingTransport) RoundTrip(r *http.Request) (*http.Response, error) { - c.n++ - return http.DefaultTransport.RoundTrip(r) -} - -// TestReachableSkipsIdentityWhenNothingAnswers pins the probe's ORDER, not just -// its verdicts. The identity read must run only when the TCP sweep already found -// the address answering — i.e. only when the node would otherwise have been -// kept, which is the sole case where the two need telling apart. -// -// Asking the other way round is what this guards against: it would spend a -// node-info timeout on every departed node, serially, inside the browser's scan -// cycle. The verdicts are identical either way, so nothing but a request count -// can catch a regression here. -func TestReachable_SkipsIdentityWhenNothingAnswers(t *testing.T) { - counter := &countingTransport{} - d := &daemon{http: &http.Client{Transport: counter, Timeout: nodeInfoFetchTimeout}} - - dead := rawNode("v=1", "uuid=gone", "ip=127.0.0.1", - fmt.Sprintf("ni=%d", closedPort(t)), fmt.Sprintf("ol=%d", closedPort(t))) - require.False(t, d.reachable(dead), "a node answering on nothing must not be kept alive") - require.Equal(t, 0, counter.n, "a departed node cost") - - // When the address DOES answer, the identity read is exactly what decides it. - host, port := nodeInfoServer(t, "someone-else") - live := rawNode("v=1", "uuid=old-uuid", "ip="+host, fmt.Sprintf("ni=%d", port)) - require.False(t, d.reachable(live), "an answering address reporting a different hostUuid must evict") - require.Equal(t, 1, counter.n, "node-info requests") -} - -// testNow is captured once so every ghost() in a run shares one base: LastSeen -// is Unix SECONDS, so computing it per call would let a wall-clock rollover -// between two calls shift the gap by a second and flip a threshold assertion. -var testNow = time.Now().Unix() - -// named builds a directory entry with an explicit hostname; ghost is the common -// case where the hostname is the one a wipe would have preserved. -func named(uuid, name, ip string, ageSeconds int64) noderec.DirectoryNode { - return noderec.DirectoryNode{ - HostUUID: uuid, - Name: name, - IP: ip, - LastSeen: testNow - ageSeconds, - } -} - -func ghost(uuid, ip string, ageSeconds int64) noderec.DirectoryNode { - return named(uuid, "wiped-host", ip, ageSeconds) -} - -// withNodeInfo advertises an ni port on a directory entry — the only service the -// supersedence proof reads. -func withNodeInfo(n noderec.DirectoryNode, port int) noderec.DirectoryNode { - n.Services = map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - } - return n -} - -func candidateUUIDs(nodes []noderec.DirectoryNode) string { - ids := make([]string, 0, len(nodes)) - for _, n := range nodes { - ids = append(ids, n.HostUUID) - } - return strings.Join(ids, ",") -} - -// TestSupersedeCandidatesNominatesGhost covers the nomination: when the wiped -// machine reappears under a new uuid at the same address and hostname, the -// record it replaced is put forward for confirmation — and left in place until -// it is confirmed. -func TestSupersedeCandidates_NominatesGhost(t *testing.T) { - d := newDirectory() - d.upsert(ghost("old-uuid", "192.168.1.10", 3600)) - - candidates := d.supersedeCandidates(ghost("new-uuid", "192.168.1.10", 0), "self-uuid") - require.Len(t, candidates, 1, "candidates") - require.Equal(t, "old-uuid", candidates[0].HostUUID, "candidates (%v)", candidates) - _, ok := d.get("old-uuid") - require.True(t, ok, "nominating a record must not delete it; only proof may") -} - -// TestSupersedeCandidatesKeepFreshNeighbours is the false-positive guard: two -// records observed at about the same time are the shape a misconfiguration -// takes, not a wipe, and are not worth a probe. -func TestSupersedeCandidates_KeepFreshNeighbourRecords(t *testing.T) { - d := newDirectory() - d.upsert(ghost("peer-a", "192.168.1.10", supersedeMinAge-1)) - - require.Empty(t, d.supersedeCandidates(ghost("peer-b", "192.168.1.10", 0), "self-uuid"), "a gap under supersedeMinAge must not nominate") -} - -// TestSupersedeCandidatesRequireSameHostname keeps the nomination narrow. Two -// machines colliding on an address almost never also collide on a hostname, so a -// name mismatch means we are not looking at one machine. The wiped-AND-renamed -// case is left to the miss-threshold path, which reaches the same proof. -func TestSupersedeCandidates_RequireMatchingHostname(t *testing.T) { - d := newDirectory() - d.upsert(named("old-uuid", "old-name", "192.168.1.10", 3600)) - - require.Empty(t, d.supersedeCandidates(named("new-uuid", "new-name", "192.168.1.10", 0), "self-uuid"), "a different hostname on the address must not be nominated") -} - -// TestSupersedeCandidatesIgnoreTheAddress covers the signal that was removed. A -// wiped machine keeps its hostname whether or not it comes back on the address -// it had before, so a stale namesake elsewhere is nominated too — and, in the -// other direction, sharing an address is no longer a reason to nominate -// anything, which is what stopped two hosts on a direct-connect link from -// putting each other forward. -func TestSupersedeCandidates_IgnoreAddressWhenComparingIdentity(t *testing.T) { - d := newDirectory() - d.upsert(ghost("peer-a", "192.168.1.10", 3600)) - - candidates := d.supersedeCandidates(ghost("peer-b", "192.168.1.11", 0), "self-uuid") - require.Len(t, candidates, 1, "candidates") - require.Equal(t, "peer-a", candidates[0].HostUUID, "candidates (%v)", candidates) - - // The arriving record must still carry an address: that is where the - // confirming read is made, so without one there is nothing to ask. - d2 := newDirectory() - d2.upsert(ghost("peer-c", "192.168.1.10", 3600)) - require.Empty(t, d2.supersedeCandidates(ghost("peer-d", "", 0), "self-uuid"), "an unaddressable arrival can prove nothing") -} - -// TestSupersedeCandidatesNeverIncludeSelf protects the local card. Self is -// registry-driven, so its LastSeen advances only when this node republishes and -// can legitimately look stale — it must not be displaceable by a peer claiming -// its address. Evicting the local node is a failure mode this daemon has hit -// before (see the self-guard in onBrowse). -func TestSupersedeCandidates_NeverIncludeSelf(t *testing.T) { - d := newDirectory() - d.upsert(ghost("self-uuid", "192.168.1.10", 3600)) - - require.Empty(t, d.supersedeCandidates(ghost("peer-uuid", "192.168.1.10", 0), "self-uuid"), "self must never be nominated by a peer") - - // Nor may the local record nominate peers on its way in. - d2 := newDirectory() - d2.upsert(ghost("peer-uuid", "192.168.1.10", 3600)) - require.Empty(t, d2.supersedeCandidates(ghost("self-uuid", "192.168.1.10", 0), "self-uuid"), "self must not nominate peers") -} - -// TestSupersedeCandidatesAreDeterministic pins the order, since the caller turns -// the confirmed subset into a stream of node-removed notifications. -func TestSupersedeCandidates_AreDeterministic(t *testing.T) { - d := newDirectory() - d.upsert(ghost("uuid-c", "192.168.1.10", 3600)) - d.upsert(ghost("uuid-a", "192.168.1.10", 7200)) - d.upsert(ghost("uuid-b", "192.168.1.10", 5400)) - - require.Equal(t, "uuid-a,uuid-b,uuid-c", candidateUUIDs(d.supersedeCandidates(ghost("uuid-live", "192.168.1.10", 0), "self-uuid")), "candidate order") -} - -// TestUpsertEvictingRejudgesRecordsThatMoved covers the gap the proof opens: it -// is a network read, so the directory is unlocked while it runs and the entry -// under judgement can change underneath it. A verdict formed against a state -// that no longer exists must not be applied to the state that replaced it. -func TestUpsertEvicting_RechecksRecordsWhoseAddressChanged(t *testing.T) { - d := newDirectory() - judged := named("old-uuid", "wiped-host", "192.168.1.10", 3600) - d.upsert(judged) - - moved := judged - moved.IP = "192.168.1.11" - d.upsert(moved) - - arriving := named("new-uuid", "wiped-host", "192.168.1.10", 0) - require.Empty(t, d.upsertEvicting(arriving, []noderec.DirectoryNode{judged}), "a record that changed during the probe must be judged again, not evicted") - _, ok := d.get("old-uuid") - require.True(t, ok, "the re-addressed record must survive") - _, ok = d.get("new-uuid") - require.True(t, ok, "the arriving node must be stored either way") -} - -// TestSupersedingUpsertRequiresIdentityProof is the security guard. Address and -// hostname come from an unauthenticated mDNS record and LastSeen freezes for a -// healthy peer exactly as it does for a ghost, so collapsing on those alone -// would let anything on the LAN delete a live node by claiming its address — -// and the browser re-reports a node only when its record CHANGES, so the peer -// might never come back. Nothing is deleted without an answer from the machine. -func TestSupersedingUpsert_RequiresIdentityProof(t *testing.T) { - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - // Nothing is listening on the record's own ni port, so it cannot speak for - // itself and the claimant's answer is the only evidence in play. - d.dir.upsert(withNodeInfo(named("live-uuid", "wiped-host", loopbackHost, 3600), closedPort(t))) - claim := named("claimant-uuid", "wiped-host", loopbackHost, 0) - - // Nothing to ask: the claimant advertises no node-info port. - d.supersedingUpsert(claim) - _, ok := d.dir.get("live-uuid") - require.True(t, ok, "an unprovable claim must never evict a peer: mDNS is not a removal primitive") - - // Asked, but unanswerable — the same verdict. - d.supersedingUpsert(withNodeInfo(claim, closedPort(t))) - _, ok = d.dir.get("live-uuid") - require.True(t, ok, "an address that cannot answer proves nothing; the peer must survive") - - // Answered, and the machine is still the node the record describes. - _, alivePort := nodeInfoServer(t, "live-uuid") - d.supersedingUpsert(withNodeInfo(claim, alivePort)) - _, ok = d.dir.get("live-uuid") - require.True(t, ok, "node-info confirming the record's own identity must keep it") - - // Only a definite mismatch collapses the pair. - _, wipedPort := nodeInfoServer(t, "claimant-uuid") - d.supersedingUpsert(withNodeInfo(claim, wipedPort)) - _, ok = d.dir.get("live-uuid") - require.False(t, ok, "node-info naming a different host is proof; the superseded record must go") - _, ok = d.dir.get("claimant-uuid") - require.True(t, ok, "the arriving node must be stored") -} - -// TestSupersedingUpsertKeepsARecordThatAnswersForItself is the case that made -// the address a liability. Two identically-provisioned machines wired back to -// back both advertise the link's address, so each looked like the other's ghost; -// the pair is now told apart by asking the older record's own endpoint who it -// is. A record that names itself is a live peer, not a leftover, whatever -// address the arriving node claims. -func TestSupersedingUpsert_KeepsARecordThatAnswersForItself(t *testing.T) { - host, livePort := nodeInfoServer(t, "peer-a") - _, claimPort := nodeInfoServer(t, "peer-b") - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - d.dir.upsert(withNodeInfo(named("peer-a", "dgx-station", host, 3600), livePort)) - - d.supersedingUpsert(withNodeInfo(named("peer-b", "dgx-station", host, 0), claimPort)) - - _, ok := d.dir.get("peer-a") - require.True(t, ok, "a record still answering with its own hostUuid is a live peer sharing an address, not a ghost") - _, ok = d.dir.get("peer-b") - require.True(t, ok, "the arriving node must be stored") -} - -// TestSupersedingUpsertKeepsAnUnaskableRecord covers the other half of that -// defence: with no ni port to ask, a live namesake and a leftover look -// identical, so the record stays and the miss-threshold path reaches it when it -// stops advertising. -func TestSupersedingUpsert_KeepsAnUnaskableRecord(t *testing.T) { - host, claimPort := nodeInfoServer(t, "claimant-uuid") - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - d.dir.upsert(named("live-uuid", "wiped-host", host, 3600)) - - d.supersedingUpsert(withNodeInfo(named("claimant-uuid", "wiped-host", host, 0), claimPort)) - - _, ok := d.dir.get("live-uuid") - require.True(t, ok, "a record that cannot be asked who it is must never be evicted here") -} - -// TestOnBrowseEmitsSupersededRemoval covers the wiring: the daemon must tell the -// broker the ghost is gone, and drop its cached enrichment so a later -// rediscovery under that uuid re-probes instead of resurrecting stale metrics. -func TestOnBrowse_EmitsSupersededRemoval(t *testing.T) { - host, port := nodeInfoServer(t, "new-uuid") - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - d.dir.upsert(withNodeInfo(ghost("old-uuid", host, 3600), closedPort(t))) - d.lastInfo["old-uuid"] = NodeInfoResponse{HostUUID: "old-uuid"} - - d.onBrowse(DiscoveryEvent{ - Type: "discovered", - Node: RawNode{ - ID: "wiped-host", - Addresses: []string{host}, - TXT: []string{"v=1", "uuid=new-uuid", "ip=" + host, fmt.Sprintf("ni=%d", port)}, - }, - }) - - _, ok := d.dir.get("old-uuid") - require.False(t, ok, "onBrowse must collapse the record the arriving node is proven to replace") - _, ok = d.dir.get("new-uuid") - require.True(t, ok, "the arriving node must be stored") - d.infoMu.Lock() - assert.NotContains(t, d.lastInfo, "old-uuid", "a superseded node's enrichment cache must be dropped") - d.infoMu.Unlock() -} diff --git a/services/nvpair-node-scanner/main.go b/services/nvpair-node-scanner/main.go index 9c8d9b09a..b23ed954e 100644 --- a/services/nvpair-node-scanner/main.go +++ b/services/nvpair-node-scanner/main.go @@ -20,10 +20,7 @@ import ( func main() { ipcPath := flag.String("ipc", "", "IPC endpoint: Unix domain socket path or Windows named pipe (default: stdin/stdout)") - clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); when set, browsed peers holding a pin we trust are annotated trusted in the directory") - clientCert := flag.String("client-cert", "", "path to PEM client certificate to present when fetching /v1/node-info from TLS-enabled nodes (requires --client-key; node-info enrichment stays plain HTTP if unset)") - clientKey := flag.String("client-key", "", "path to PEM client private key matching --client-cert") - caBundle := flag.String("ca-bundle", "", "path to PEM CA bundle to trust for verifying node-info server certificates (additive to the system trust store); enabling any of these TLS flags moves node-info enrichment onto HTTPS") + clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); authenticates remote inventory using exact peer pins") showVersion := flag.Bool("version", false, "print version and exit") resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo) flag.Parse() @@ -35,26 +32,10 @@ func main() { applog.Init("nvpair-node-scanner", resolveLevel()) - // While this node is a cluster member, the daemon advertises its cluster - // principal as cluster-uuid= and annotates a browsed peer as trusted when it - // holds a pin for that peer's principal. The mesh is a live view of the - // cluster dir, so a create/join/leave that happens while this process runs is - // reflected in what this node advertises (see reloadIdentity). + // Membership controls local advertisement; a peer pin authorizes an + // authentication attempt rather than trusting its announcement. mesh := clustertrust.Open(*clusterDir) - // Optional, dormant-by-default: with none of these flags set, buildTLSClient - // returns nil and node-info enrichment stays plain HTTP (the standard path). - // Setting them moves node-info fetches onto HTTPS — scaffolding for a future - // TLS node-info rollout, gated only on operator flags (never per-node). - tlsOpts := tlsClientOptions{CertPath: *clientCert, KeyPath: *clientKey, CABundlePath: *caBundle} - if err := tlsOpts.validate(); err != nil { - log.Fatalf("invalid TLS configuration: %v", err) - } - tlsHTTP, err := buildTLSClient(tlsOpts, nodeInfoFetchTimeout) - if err != nil { - log.Fatalf("failed to build node-info TLS client: %v", err) - } - var transport io.ReadWriteCloser if *ipcPath != "" { conn, err := dialIPC(*ipcPath) @@ -84,7 +65,7 @@ func main() { }() codec := NewCodec(transport) - scanner := NewScanner(codec, mesh, *clusterDir, tlsHTTP) + scanner := NewScanner(codec, mesh, *clusterDir) if err := scanner.Run(ctx); err != nil && ctx.Err() == nil { log.Fatalf("scanner error: %v", err) diff --git a/services/nvpair-node-scanner/node_info_gate.go b/services/nvpair-node-scanner/node_info_gate.go index 4996f58cc..269b45680 100644 --- a/services/nvpair-node-scanner/node_info_gate.go +++ b/services/nvpair-node-scanner/node_info_gate.go @@ -23,7 +23,31 @@ type nodeInfoOriginGate struct { } func (g *nodeInfoOriginGate) acquire(ctx context.Context, origin string) (func(), bool) { + permit := g.permit(origin) + select { + case permit <- struct{}{}: + return func() { <-permit }, true + case <-ctx.Done(): + return nil, false + } +} + +// tryAcquire defers candidate authentication while an origin is busy, preventing +// copied announcements from queuing ahead of the verified peer's telemetry. +func (g *nodeInfoOriginGate) tryAcquire(origin string) (func(), bool) { + permit := g.permit(origin) + select { + case permit <- struct{}{}: + return func() { <-permit }, true + default: + return nil, false + } +} + +// permit returns the shared slot without holding the map lock while a caller waits. +func (g *nodeInfoOriginGate) permit(origin string) chan struct{} { g.mu.Lock() + defer g.mu.Unlock() permit := g.origins[origin] if permit == nil { if g.origins == nil { @@ -32,12 +56,5 @@ func (g *nodeInfoOriginGate) acquire(ctx context.Context, origin string) (func() permit = make(chan struct{}, 1) g.origins[origin] = permit } - g.mu.Unlock() - - select { - case permit <- struct{}{}: - return func() { <-permit }, true - case <-ctx.Done(): - return nil, false - } + return permit } diff --git a/services/nvpair-node-scanner/node_info_outage_test.go b/services/nvpair-node-scanner/node_info_outage_test.go index 6559ecd5d..099e33d2a 100644 --- a/services/nvpair-node-scanner/node_info_outage_test.go +++ b/services/nvpair-node-scanner/node_info_outage_test.go @@ -6,20 +6,10 @@ package main import ( "bytes" "context" - "encoding/json" "log/slog" - "net" - "net/http" - "net/http/httptest" - "net/url" - "strconv" "strings" - "sync" - "sync/atomic" "testing" - "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -71,79 +61,6 @@ func TestNodeInfoOutage_EmitsNothingWhileAvailabilityIsUnchanged(t *testing.T) { require.Empty(t, logs.Bytes(), "a healthy peer must log nothing") } -// TestNodeInfoConnectGivesUpBeforeTheRequestBudget covers the split itself. The -// address is reserved for documentation and is routed nowhere, so connecting to -// it either fails immediately (the local network says so) or hangs — and hanging -// is the case that mattered: it must end at the connect budget rather than -// consume the whole request timeout the enrichment client allows. -func TestNodeInfoConnect_GivesUpBeforeTheRequestBudget(t *testing.T) { - require.Less(t, nodeInfoDialTimeout, nodeInfoFetchTimeout, "a connect budget of") - - start := time.Now() - conn, err := nodeInfoTransport(nil).DialContext(context.Background(), "tcp", "192.0.2.1:14300") - if err == nil { - require.NoError(t, conn.Close()) - require.FailNow(t, "nothing may answer at a documentation address") - } - require.Less(t, time.Since(start), nodeInfoFetchTimeout, "connect must give up at the dial budget %v", nodeInfoDialTimeout) -} - -func TestNodeInfoTransport_BoundsConnectionsPerHost(t *testing.T) { - transport := nodeInfoTransport(nil) - require.Equal(t, 1, transport.MaxConnsPerHost) - require.Equal(t, 1, transport.MaxIdleConnsPerHost) -} - -// Two requests share one connection and each fits its own timeout. Time spent -// waiting for that connection must not consume the second request's response budget. -func TestNodeInfoFetches_UseIndependentRequestBudgets(t *testing.T) { - const responseDelay = 200 * time.Millisecond - var requests atomic.Int32 - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests.Add(1) - time.Sleep(responseDelay) - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"})) - })) - var connections atomic.Int32 - server.Config.ConnState = func(_ net.Conn, state http.ConnState) { - if state == http.StateNew { - connections.Add(1) - } - } - server.Start() - defer server.Close() - - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - transport := nodeInfoTransport(nil) - defer transport.CloseIdleConnections() - d := &daemon{http: &http.Client{Timeout: 300 * time.Millisecond, Transport: transport}} - - start := make(chan struct{}) - results := make(chan bool, 2) - var wg sync.WaitGroup - for range 2 { - wg.Add(1) - go func() { - defer wg.Done() - <-start - _, ok := d.fetchNodeInfo(serverURL.Hostname(), port) - results <- ok - }() - } - close(start) - wg.Wait() - close(results) - - for ok := range results { - require.True(t, ok, "a queued healthy request consumed its timeout before reaching the peer") - } - require.Equal(t, int32(2), requests.Load(), "server received") - require.Equal(t, int32(1), connections.Load(), "requests used") -} - func TestNodeInfoOriginGate_HonorsCanceledWaiter(t *testing.T) { var gate nodeInfoOriginGate release, acquired := gate.acquire(context.Background(), "http://127.0.0.1:14318") @@ -158,39 +75,3 @@ func TestNodeInfoOriginGate_HonorsCanceledWaiter(t *testing.T) { require.FailNow(t, "canceled origin request acquired the occupied gate") } } - -func TestNodeInfoErrorBody_IsDrainedForConnectionReuse(t *testing.T) { - var requests atomic.Int32 - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if requests.Add(1) == 1 { - w.WriteHeader(http.StatusServiceUnavailable) - _, err := w.Write([]byte("temporarily unavailable")) - assert.NoError(t, err) - return - } - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"})) - })) - var connections atomic.Int32 - server.Config.ConnState = func(_ net.Conn, state http.ConnState) { - if state == http.StateNew { - connections.Add(1) - } - } - server.Start() - defer server.Close() - - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - transport := nodeInfoTransport(nil) - defer transport.CloseIdleConnections() - d := &daemon{http: &http.Client{Timeout: time.Second, Transport: transport}} - - _, ok := d.fetchNodeInfo(serverURL.Hostname(), port) - require.False(t, ok, "HTTP 503 unexpectedly succeeded") - info, ok := d.fetchNodeInfo(serverURL.Hostname(), port) - require.True(t, ok, "second fetch (%v, %v)", info, ok) - require.Equal(t, "peer-uuid", info.HostUUID, "second fetch (%v, %v)", info, ok) - require.Equal(t, int32(1), connections.Load(), "requests used") -} diff --git a/services/nvpair-node-scanner/refresh_test.go b/services/nvpair-node-scanner/refresh_test.go index 78518fd23..4e4c9159d 100644 --- a/services/nvpair-node-scanner/refresh_test.go +++ b/services/nvpair-node-scanner/refresh_test.go @@ -4,11 +4,11 @@ package main import ( + "context" "encoding/json" "net/http" "net/http/httptest" "net/url" - "strconv" "sync" "testing" @@ -94,7 +94,6 @@ func newRefreshTestDaemon() *daemon { reg: newRegistry("self-node-uuid", "", selfAddrs("127.0.0.1")), dir: newDirectory(), modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, - lastInfo: make(map[string]NodeInfoResponse), lastInfoAt: make(map[string]time.Time), lastModels: make(map[string][]string), lastModelsByEngine: make(map[string]map[string][]string), @@ -179,9 +178,89 @@ func TestRefreshSelf_DialsLoopback(t *testing.T) { assert.Equal(t, unreachableLAN, got.IP, "self IP (%v)", unreachableLAN) } -// TestRefreshPullDeleteAndUnchanged covers three required behaviors together: a -// pull and a delete each change the inventory and emit exactly one update, while -// an unchanged inventory emits nothing (no repeated updates). +// Republishing local services during an outage must retain the latest periodic inventory. +func TestPublishSelf_RetainsLatestModelsAfterLocalInventoryFailure(t *testing.T) { + d := newSelfTestDaemon("self-uuid", "192.0.2.1") + stub, port := startModelsStub(t, http.StatusOK, map[string]any{ + "models": []string{"initial-model"}, + "modelsByEngine": map[string][]string{"ollama": {"initial-model"}}, + "loadedByEngine": map[string][]string{"ollama": {"initial-model"}}, + }) + d.reg.register(noderec.RegisterParams{Service: noderec.ServiceEngineManager, Port: port}) + d.publishSelf() + stub.set(http.StatusOK, map[string]any{ + "models": []string{"latest-model"}, + "modelsByEngine": map[string][]string{"ollama": {"latest-model"}}, + "loadedByEngine": map[string][]string{"ollama": {"latest-model"}}, + }) + require.True(t, d.refreshNodeModels("self-uuid", "192.0.2.1", port, "")) + + stub.set(http.StatusServiceUnavailable, nil) + d.publishSelf() + + node, present := d.dir.get("self-uuid") + require.True(t, present) + assert.Equal(t, []string{"latest-model"}, node.Models) + assert.Equal(t, map[string][]string{"ollama": {"latest-model"}}, node.ModelsByEngine) + assert.Equal(t, map[string][]string{"ollama": {"latest-model"}}, node.LoadedByEngine) +} + +// Publishing local services must not wait for an engine-manager read before +// owned hardware notifications and subsequent IPC can proceed. +func TestPublishSelf_UsesCachedModelsWithoutWaitingForHTTP(t *testing.T) { + d := newSelfTestDaemon("self-uuid", "192.0.2.1") + blockReads := make(chan struct{}) + readStarted := make(chan struct{}, 1) + release := make(chan struct{}) + releaseRead := sync.OnceFunc(func() { close(release) }) + defer releaseRead() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + select { + case <-blockReads: + readStarted <- struct{}{} + <-release + default: + } + assert.NoError(t, json.NewEncoder(w).Encode(map[string][]string{"models": {"retained-model"}})) + })) + t.Cleanup(server.Close) + port := portFromURL(t, server.URL) + d.reg.register(noderec.RegisterParams{Service: noderec.ServiceEngineManager, Port: port}) + d.publishSelf() + require.True(t, d.refreshNodeModels("self-uuid", "192.0.2.1", port, "")) + select { + case <-d.modelsWake: + default: + } + close(blockReads) + + published := make(chan struct{}) + go func() { + d.publishSelf() + close(published) + }() + select { + case <-published: + case <-readStarted: + t.Error("local publication started an HTTP read instead of using cached models") + releaseRead() + <-published + case <-time.After(5 * time.Second): + releaseRead() + t.Fatal("local publication did not complete") + } + + node, present := d.dir.get("self-uuid") + require.True(t, present) + assert.Equal(t, []string{"retained-model"}, node.Models) + select { + case <-d.modelsWake: + default: + t.Error("local publication must schedule model refresh") + } +} + +// Downloads and deletions each change inventory once; unchanged reads stay quiet. func TestRefreshModels_ReportsDownloadsAndDeletionsButNotUnchangedInventory(t *testing.T) { d := newRefreshTestDaemon() stub, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -262,12 +341,6 @@ func TestRefreshFetchFailure_RetainsInventory(t *testing.T) { got, _ := d.dir.get("peer-C") assert.Equal(t, []string{"keep-me"}, got.Models, "Models after failure") assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, got.ModelsByEngine, "ModelsByEngine after failure") - d.infoMu.Lock() - cachedFlat := d.lastModels["peer-C"] - cachedByEngine := d.lastModelsByEngine["peer-C"] - d.infoMu.Unlock() - assert.Equal(t, []string{"keep-me"}, cachedFlat, "cached flat inventory") - assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, cachedByEngine, "cached per-engine inventory") } // TestRefreshPartialModelsByEngine covers requirement 4: a partial @@ -351,10 +424,6 @@ func TestRefreshModels_RetainsLoadedModelsAfterFetchFailure(t *testing.T) { assert.False(t, d.refreshNodeModels("peer-N", "127.0.0.1", port, ""), "a failed fetch should not report a change") got, _ := d.dir.get("peer-N") assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, got.LoadedByEngine, "LoadedByEngine after failure") - d.infoMu.Lock() - cachedLoaded := d.lastLoadedByEngine["peer-N"] - d.infoMu.Unlock() - assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, cachedLoaded, "cached loaded inventory") } // TestApplyModelsGuard covers requirement 6 at the directory layer: a completed @@ -414,44 +483,49 @@ func TestRefreshNodeModels_DoesNotRestoreNodeRemovedDuringSweep(t *testing.T) { // either is released. A serial sweep would block inside the first handler and // never enter the second, so the barrier wait fails fast. The time.After bounds // are failure ceilings, not success-timing assertions, so the test isn't flaky. -func TestRefreshOnce_PollsSlowPeersConcurrently(t *testing.T) { - const peers = 2 // must stay <= modelsRefreshConcurrency - entered := make(chan struct{}, peers) - release := make(chan struct{}) - handler := func(w http.ResponseWriter, r *http.Request) { - entered <- struct{}{} - <-release // hold until both peers are concurrently in-flight - assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"models": []string{"m"}})) +func TestRefreshModels_ConcurrentAuthenticatedPeers(t *testing.T) { + first := newDiscoveryPeerFixture(t, "peer-1") + first.discover(t) + second := newDiscoveryPeerFixture(t, "peer-2") + second.attach(t, first.daemon) + entered, release := make(chan struct{}, 2), make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + for _, f := range []*discoveryPeerFixture{first, second} { + f.info.Peer.Services[noderec.ServiceEngineManager] = portFromURL(t, f.server.URL) + f.daemon.refreshTelemetryOnce(context.Background()) + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + entered <- struct{}{} + <-release + inventory := modelInventoryResponse{Models: []string{"m"}} + assert.NoError(t, json.NewEncoder(w).Encode(inventory)) + return true + } } - s1 := httptest.NewServer(http.HandlerFunc(handler)) - s2 := httptest.NewServer(http.HandlerFunc(handler)) - t.Cleanup(s1.Close) - t.Cleanup(s2.Close) - - d := newRefreshTestDaemon() - seedEMNode(d, "peer-1", "127.0.0.1", portFromURL(t, s1.URL)) - seedEMNode(d, "peer-2", "127.0.0.1", portFromURL(t, s2.URL)) - done := make(chan struct{}) - go func() { d.refreshModelsOnce(); close(done) }() - - // Both handlers must be entered before either is released; only a concurrent - // sweep can reach this point. - for i := 0; i < peers; i++ { + go func() { first.daemon.refreshModelsOnce(); close(done) }() + for range 2 { select { case <-entered: - case <-time.After(5 * time.Second): - require.FailNowf(t, "model sweep did not issue concurrent requests", "%d of %d peers entered", i, peers) + case <-time.After(2 * time.Second): + require.FailNow(t, "models did not refresh peers concurrently") } } - close(release) + releaseOnce.Do(func() { close(release) }) select { case <-done: - case <-time.After(5 * time.Second): - require.FailNow(t, "refreshModelsOnce did not complete after releasing peers") + case <-time.After(2 * time.Second): + require.FailNow(t, "models refresh did not finish") + } + for _, uuid := range []string{"peer-1", "peer-2"} { + node, _ := first.daemon.dir.get(uuid) + assert.Equal(t, []string{"m"}, node.Models, "peer %s", uuid) } - got, _ := d.dir.get("peer-1") - assert.Equal(t, []string{"m"}, got.Models, "peer-1 Models") - got, _ = d.dir.get("peer-2") - assert.Equal(t, []string{"m"}, got.Models, "peer-2 Models") +} + +type modelInventoryResponse struct { + Models []string `json:"models"` } diff --git a/services/nvpair-node-scanner/scanner.go b/services/nvpair-node-scanner/scanner.go index 8450ed379..576f044b0 100644 --- a/services/nvpair-node-scanner/scanner.go +++ b/services/nvpair-node-scanner/scanner.go @@ -9,7 +9,6 @@ import ( "io" "log" "log/slog" - "net/http" "nvpair-shared/applog" "nvpair-shared/clustertrust" @@ -49,16 +48,12 @@ type Scanner struct { cancel context.CancelFunc } -// NewScanner builds the scanner around the promoted daemon. mesh (non-nil when -// this node is clustered) is threaded to the daemon for the trust annotation on -// browsed nodes; clusterDir lets the daemon resolve its node identity from the -// same base cluster-manager persists to (empty selects the shared default). -// tlsHTTP is nil unless the operator opted node-info enrichment onto HTTPS via -// the --client-cert/--client-key/--ca-bundle flags. -func NewScanner(codec *Codec, mesh *clustertrust.Mesh, clusterDir string, tlsHTTP *http.Client) *Scanner { +// NewScanner uses the live mesh for authenticated discovery. clusterDir also +// selects the local identity root shared with cluster-manager. +func NewScanner(codec *Codec, mesh *clustertrust.Mesh, clusterDir string) *Scanner { return &Scanner{ codec: codec, - daemon: newDaemon(codec, mesh, clusterDir, tlsHTTP), + daemon: newDaemon(codec, mesh, clusterDir), } } diff --git a/services/nvpair-node-scanner/spec.md b/services/nvpair-node-scanner/spec.md index 3c7757567..4645acc58 100644 --- a/services/nvpair-node-scanner/spec.md +++ b/services/nvpair-node-scanner/spec.md @@ -97,11 +97,13 @@ records and may read public hardware measurements separately for display. - The local host, verified peers, unverified discoveries, and manually configured servers have separate records. A claimed node identifier, a UUID, cannot cause unverified data to be merged into a local or verified peer record. -- Scanner refreshes inventory every 15 seconds and reads healthy node-info +- Scanner refreshes models every 15 seconds and reads healthy node-info servers every two seconds with stable per-node jitter, a small fixed timing offset that spreads requests across nodes. Failed remote reads use backoff, increasing the wait: 4, 8, 16, then 30 seconds between retries, returning to the - normal interval on success. Local reads stay at the normal interval. + normal interval on success. Each accepted peer response updates metadata, + hardware inventory, and scheduling telemetry together. Local hardware arrives + over owned worker IPC at the normal interval. - Scanner retains the last successfully read inventory during temporary failures. Samples retain their validity and age, including time spent in requests, queues, and caches. Forwarding or resending old data must not make it look new. @@ -117,7 +119,7 @@ records and may read public hardware measurements separately for display. authenticated inventory reads or inference responses, not just an open TCP port or unverified network claims. If a verified peer remains after its last discovery expires, Scanner continues - checking availability during inventory refresh. After a minute without + checking availability through the same canonical sampling loop. After a minute without authenticated inventory or inference activity, Scanner removes its record and caches while preserving membership and saved certificates. - Scanner applies trust changes while running. Removed or replaced pins clear @@ -368,8 +370,10 @@ Scanner reads a peer's models using the verified engine-manager (`em`) service details and `GET /v1/models` over pinned mTLS. For the local host, Scanner uses its own registry and unencrypted HTTP over loopback. It adds the combined model list, `modelsByEngine`, and `loadedByEngine` to the directory, retaining the last -successfully read data during temporary failures. A newly authenticated peer -schedules a model refresh without waiting for the next periodic inventory sweep. +successfully read data during temporary failures. Local publication uses retained +models without waiting for HTTP while holding the local identity lock. A local +registration or newly authenticated peer schedules the existing model sweep +without waiting for its next periodic run. Scanner compares model lists without depending on their order. A present engine entry with an empty list reports no models for that engine; a missing entry means @@ -559,7 +563,7 @@ sequenceDiagram ``` The diagram shows separate display and service paths. HTTP display readings -never flow into scheduler or verified peer records. Service inventory refreshes -every 15 seconds and measurements every two seconds, independently of mDNS -changes. The broker forwards trust changes to Scanner and clears affected +never flow into scheduler or verified peer records. Models refresh every 15 seconds; +peer metadata, hardware, and measurements share the two-second sampling loop, +independently of mDNS changes. The broker forwards trust changes to Scanner and clears affected verification and service samples when membership or pins change. diff --git a/services/nvpair-node-scanner/telemetry_test.go b/services/nvpair-node-scanner/telemetry_test.go index 897a84657..0866abb4c 100644 --- a/services/nvpair-node-scanner/telemetry_test.go +++ b/services/nvpair-node-scanner/telemetry_test.go @@ -8,9 +8,6 @@ import ( "context" "encoding/json" "net/http" - "net/http/httptest" - "net/url" - "strconv" "sync" "sync/atomic" "testing" @@ -22,118 +19,151 @@ import ( "nvpair-shared/noderec" ) -func TestRefreshNodeTelemetry_EmitsFreshnessAndUtilization(t *testing.T) { - response := NodeInfoResponse{ - GPUs: []GPUInfo{ - {Name: "GPU 0", UtilizationPercent: 0}, - {Name: "GPU 1", UtilizationPercent: 84}, - }, - TelemetryValid: true, - MSSince: 137, - HostUUID: "node-a", +// One authenticated read must update the peer's metadata and hardware and send +// that same sample to scheduling, rather than waiting for another inventory read. +func TestTelemetryDispatch_UpdatesInventoryAndSchedulingTogether(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.info.Peer.Name = "updated peer" + f.info.Peer.Services[noderec.ServiceOllama] = 11435 + f.info.CPU = &CPUInfo{Name: "updated CPU", Cores: 16} + f.info.GPUs[0].UtilizationPercent = 95 + var reads atomic.Int32 + started, release := make(chan struct{}), make(chan struct{}) + releaseRead := sync.OnceFunc(func() { close(release) }) + defer releaseRead() + f.respond = func(http.ResponseWriter, *http.Request) bool { + reads.Add(1) + close(started) + <-release + return false } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - assert.NoError(t, json.NewEncoder(w).Encode(response)) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - var output bytes.Buffer - d := &daemon{codec: NewCodec(&output), http: server.Client()} - require.True(t, d.refreshNodeTelemetry(context.Background(), "node-a", serverURL.Hostname(), port), "valid node-info response did not emit telemetry") - - var message Message - require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message), "decode notification") - require.Equal(t, noderec.NotifyNodeTelemetry, message.Method, "notification method") - var got noderec.NodeTelemetry - require.NoError(t, json.Unmarshal(message.Params, &got), "decode telemetry") - require.Equal(t, "node-a", got.HostUUID, "telemetry identity/validity (%v)", got) - require.True(t, got.TelemetryValid, "telemetry identity/validity (%v)", got) - require.Equal(t, uint32(84), got.GPUUtilizationPct, "GPU utilization") - require.GreaterOrEqual(t, got.MSSince, int64(137), "age") - require.Less(t, got.MSSince, int64(3_500), "age") -} - -func TestRefreshNodeTelemetry_RejectsMismatchedIdentity(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{ - HostUUID: "node-b", - TelemetryValid: true, - })) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - - var output bytes.Buffer - d := &daemon{codec: NewCodec(&output), http: server.Client()} - require.False(t, d.refreshNodeTelemetry(context.Background(), "node-a", serverURL.Hostname(), port), "mismatched host emitted telemetry") - require.Empty(t, output.Bytes(), "mismatched host wrote notification") + f.daemon.codec = NewCodec(&output) + finished := make(chan struct{}) + go func() { + f.daemon.refreshTelemetryOnce(context.Background()) + close(finished) + }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("authenticated sample request did not start") + } + releaseRead() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Fatal("authenticated sample did not finish") + } + + assert.Equal(t, int32(1), reads.Load()) + node, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.Equal(t, "updated peer", node.Name) + assert.Equal(t, 11435, node.Services[noderec.ServiceOllama].Port) + assert.Equal(t, f.info.CPU, node.CPU) + assert.Equal(t, f.info.GPUs, node.GPUs) + var telemetry noderec.NodeTelemetry + decoder := json.NewDecoder(&output) + for decoder.More() { + var message Message + require.NoError(t, decoder.Decode(&message)) + switch message.Method { + case noderec.NotifyNodeTelemetry: + require.NoError(t, json.Unmarshal(message.Params, &telemetry)) + } + } + assert.Equal(t, "peer-uuid", telemetry.HostUUID) + assert.True(t, telemetry.TelemetryValid) + assert.Equal(t, uint32(95), telemetry.GPUUtilizationPct) } -// TestRefreshTelemetryFailsOverToAnAnsweringAddress: this sweep is the only source -// of GPU pressure the scheduler gets, and it dialed the canonical address alone. -// A node whose canonical address is a link this host cannot reach therefore -// reported nothing at all, and the scheduler assigned it neutral pressure and -// scheduled it blind — while an address that answers sat unused in the same -// published list. -func TestRefreshTelemetry_FailsOverToAnAnsweringAddress(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{ - HostUUID: "node-a", - TelemetryValid: true, - GPUs: []GPUInfo{{Name: "GPU 0", UtilizationPercent: 61}}, - })) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - +func TestTelemetryDispatch_BackoffDoesNotInterruptHealthyPeer(t *testing.T) { + failing := newDiscoveryPeerFixture(t, "failing-peer") + failing.discover(t) + d := failing.daemon + healthy := newDiscoveryPeerFixture(t, "healthy-peer") + healthy.attach(t, d) + var failedReads, healthyReads atomic.Int32 + failing.respond = func(w http.ResponseWriter, _ *http.Request) bool { + failedReads.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + healthy.respond = func(http.ResponseWriter, *http.Request) bool { + healthyReads.Add(1) + return false + } var output bytes.Buffer - d := &daemon{ - codec: NewCodec(&output), - dir: newDirectory(), - // A short client timeout stands in for the daemon's own dial budget, so - // the address that never answers is written off promptly. - http: &http.Client{Timeout: 500 * time.Millisecond}, + d.codec = NewCodec(&output) + dispatch := func(at time.Time) { + var workers sync.WaitGroup + d.dispatchTelemetry(context.Background(), at, make(chan struct{}, telemetryRefreshConcurrency), &workers) + workers.Wait() + } + + startedAt := time.Now() + dispatch(startedAt) + require.Equal(t, int32(1), failedReads.Load()) + require.Equal(t, int32(1), healthyReads.Load()) + // The next normal dispatch precedes the failed peer's four-second retry. + // Advance the dispatch clock rather than waiting for the backoff in real time. + dispatch(startedAt.Add(2 * time.Second)) + + assert.Equal(t, int32(1), failedReads.Load(), "failed peer must wait for its retry deadline") + assert.Equal(t, int32(2), healthyReads.Load(), "healthy peer must keep its normal cadence") + var sampledHosts []string + decoder := json.NewDecoder(&output) + for decoder.More() { + var message Message + require.NoError(t, decoder.Decode(&message)) + if message.Method != noderec.NotifyNodeTelemetry { + continue + } + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + sampledHosts = append(sampledHosts, sample.HostUUID) + assert.True(t, sample.TelemetryValid) + assert.Equal(t, uint32(fakePeerGPUUtilization), sample.GPUUtilizationPct) } - // TEST-NET-1 (RFC 5737) is routed nowhere, so only the second published - // address can answer. - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-a", - IP: "192.0.2.1", - IPs: []string{"192.0.2.1", serverURL.Hostname()}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, - }) + assert.Equal(t, []string{"healthy-peer", "healthy-peer"}, sampledHosts) +} - d.refreshTelemetryOnce(context.Background()) +func TestTelemetryDispatch_AnnouncementUpdatePreservesBackoff(t *testing.T) { + test := func(name string, extraTXT []string) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, "failing-peer") + f.discover(t) + var reads atomic.Int32 + f.respond = func(w http.ResponseWriter, _ *http.Request) bool { + reads.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + dispatch := func(at time.Time) { + var workers sync.WaitGroup + f.daemon.dispatchTelemetry(context.Background(), at, make(chan struct{}, telemetryRefreshConcurrency), &workers) + workers.Wait() + } + startedAt := time.Now() + dispatch(startedAt) + require.Equal(t, int32(1), reads.Load()) - var message Message - require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message), "decode notification") - var got noderec.NodeTelemetry - require.NoError(t, json.Unmarshal(message.Params, &got), "decode telemetry") - require.Equal(t, "node-a", got.HostUUID, "telemetry (%v)", got) - require.Equal(t, uint32(61), got.GPUUtilizationPct, "telemetry (%v)", got) - require.True(t, got.TelemetryValid, "telemetry (%v)", got) + updated := f.raw + updated.TXT = append(append([]string(nil), f.raw.TXT...), extraTXT...) + f.daemon.onBrowse(DiscoveryEvent{Type: "updated", Node: updated}) + dispatch(startedAt.Add(2 * time.Second)) - // And the address that answered is remembered — in the memory the enrichment - // sweeps share — so the sweeps behind this one ask it alone rather than paying - // the unreachable address on every due telemetry attempt. - key := hostKey{hostUUID: "node-a", service: noderec.ServiceNodeInfo} - require.Equal(t, serverURL.Hostname(), d.enrichHosts.get(key), "remembered address") + assert.Equal(t, int32(1), reads.Load(), "an unchanged node-info target must retain its retry deadline") + }) + } + test("same announcement", nil) + test("another service port changes", []string{"er=14319"}) } func TestTelemetryIntervalForNode_IsStableAndBounded(t *testing.T) { - require.Equal(t, telemetryRefreshInterval, telemetryIntervalForNode(""), "empty identity interval") + assert.Equal(t, telemetryRefreshInterval, telemetryIntervalForNode("")) first := telemetryIntervalForNode("node-a") require.Equal(t, first, telemetryIntervalForNode("node-a"), "node jitter must be stable") minimum := telemetryRefreshInterval - telemetryRefreshJitter @@ -214,446 +244,3 @@ func TestTelemetryRetryGateRemoveRediscover_PreservesActiveClaim(t *testing.T) { _, ok = gate.claim("node-a", targetKey, now) require.True(t, ok, "removed attempt's late failure backed off the re-discovered node") } - -// A failed peer's retry delay belongs to that peer alone. Skipping it must not -// suppress the same refresh pass's request to a healthy node. -func TestRefreshTelemetry_SkipsBackedOffPeerWithoutDelayingHealthyPeer(t *testing.T) { - var backedOffCalls atomic.Int32 - backedOff := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - backedOffCalls.Add(1) - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-backed-off"})) - })) - defer backedOff.Close() - backedOffURL, err := url.Parse(backedOff.URL) - require.NoError(t, err, "parse backed-off server URL") - backedOffPort, err := strconv.Atoi(backedOffURL.Port()) - require.NoError(t, err, "parse backed-off server port") - - var healthyCalls atomic.Int32 - healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - healthyCalls.Add(1) - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-healthy"})) - })) - defer healthy.Close() - healthyURL, err := url.Parse(healthy.URL) - require.NoError(t, err, "parse healthy server URL") - healthyPort, err := strconv.Atoi(healthyURL.Port()) - require.NoError(t, err, "parse healthy server port") - - var output bytes.Buffer - d := &daemon{ - codec: NewCodec(&output), - dir: newDirectory(), - http: &http.Client{Timeout: time.Second}, - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-backed-off", - IP: backedOffURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: backedOffPort}, - }, - }) - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-healthy", - IP: healthyURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: healthyPort}, - }, - }) - now := time.Now() - targetKey := telemetryTargetKey([]string{backedOffURL.Hostname()}, backedOffPort) - token, ok := d.telemetryRetries.claim("node-backed-off", targetKey, now) - require.True(t, ok, "could not seed backed-off peer") - d.telemetryRetries.finish("node-backed-off", token, false, now) - - d.refreshTelemetryOnce(context.Background()) - - require.Equal(t, int32(0), backedOffCalls.Load(), "backed-off peer received") - require.Equal(t, int32(1), healthyCalls.Load(), "healthy peer received") -} - -// Changing peer-controlled TXT metadata does not create a new endpoint. -// It must not reset failure backoff and let an unhealthy peer trigger extra polls. -func TestBrowseTXTUpdate_PreservesTelemetryRetry(t *testing.T) { - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - calls.Add(1) - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"})) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - - d := newSelfTestDaemon("self-uuid", "127.0.0.1") - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "peer-uuid", - IP: serverURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, - }) - now := time.Now() - targetKey := telemetryTargetKey([]string{serverURL.Hostname()}, port) - token, ok := d.telemetryRetries.claim("peer-uuid", targetKey, now) - require.True(t, ok, "could not seed peer retry state") - d.telemetryRetries.finish("peer-uuid", token, false, now) - - d.onBrowse(DiscoveryEvent{ - Type: "updated", - Node: RawNode{ - ID: "peer", - Addresses: []string{serverURL.Hostname()}, - TXT: []string{ - "v=1", - "uuid=peer-uuid", - "ip=" + serverURL.Hostname(), - "ni=" + strconv.Itoa(port), - "peer-controlled-field=changed", - }, - }, - }) - - require.Equal(t, int32(1), calls.Load(), "browse enrichment made") - _, ok = d.telemetryRetries.claim("peer-uuid", targetKey, now) - require.False(t, ok, "TXT-only update cleared telemetry backoff for an unchanged endpoint") -} - -func TestBrowseRemoval_ClearsTelemetryRetry(t *testing.T) { - d := newSelfTestDaemon("self-uuid", "127.0.0.1") - now := time.Now() - host := "192.0.2.81" - port := 14318 - targetKey := telemetryTargetKey([]string{host}, port) - token, ok := d.telemetryRetries.claim("peer-uuid", targetKey, now) - require.True(t, ok, "could not seed peer retry state") - d.telemetryRetries.finish("peer-uuid", token, false, now) - - d.onBrowse(DiscoveryEvent{ - Type: "removed", - Node: RawNode{ - ID: "peer", - Addresses: []string{host}, - TXT: []string{ - "v=1", - "uuid=peer-uuid", - "ip=" + host, - "ni=" + strconv.Itoa(port), - }, - }, - }) - - _, ok = d.telemetryRetries.claim("peer-uuid", targetKey, now) - require.True(t, ok, "removed peer endpoint remained backed off") -} - -// The old request completes after discovery has installed a replacement endpoint. -// Its result must neither back off the replacement nor permit overlapping polls. -func TestBrowseEndpointUpdate_WaitsForOldTelemetryBeforeClaimingReplacement(t *testing.T) { - newEntered := make(chan struct{}, 1) - releaseNew := make(chan struct{}, 1) - newServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - newEntered <- struct{}{} - <-releaseNew - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"})) - })) - oldEntered := make(chan struct{}, 1) - releaseOld := make(chan struct{}, 1) - oldServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - oldEntered <- struct{}{} - <-releaseOld - w.WriteHeader(http.StatusServiceUnavailable) - })) - release := func(ch chan<- struct{}) { - select { - case ch <- struct{}{}: - default: - } - } - defer func() { - release(releaseNew) - release(releaseOld) - newServer.Close() - oldServer.Close() - }() - waitFor := func(ch <-chan struct{}, what string) { - select { - case <-ch: - case <-time.After(time.Second): - require.FailNowf(t, "timed out waiting for telemetry", "%s", what) - } - } - - newURL, err := url.Parse(newServer.URL) - require.NoError(t, err, "parse new server URL") - newPort, err := strconv.Atoi(newURL.Port()) - require.NoError(t, err, "parse new server port") - oldURL, err := url.Parse(oldServer.URL) - require.NoError(t, err, "parse old server URL") - oldPort, err := strconv.Atoi(oldURL.Port()) - require.NoError(t, err, "parse old server port") - - d := newSelfTestDaemon("self-uuid", "127.0.0.1") - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "peer-uuid", - IP: oldURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: oldPort}, - }, - }) - - browseDone := make(chan struct{}) - go func() { - d.onBrowse(DiscoveryEvent{ - Type: "updated", - Node: RawNode{ - ID: "peer", - Addresses: []string{newURL.Hostname()}, - TXT: []string{ - "v=1", - "uuid=peer-uuid", - "ip=" + newURL.Hostname(), - "ni=" + strconv.Itoa(newPort), - }, - }, - }) - close(browseDone) - }() - waitFor(newEntered, "replacement endpoint enrichment") - - telemetryDone := make(chan struct{}) - go func() { - d.refreshTelemetryOnce(context.Background()) - close(telemetryDone) - }() - waitFor(oldEntered, "old endpoint telemetry") - - release(releaseNew) - waitFor(browseDone, "directory update") - replacementTarget := telemetryTargetKey([]string{newURL.Hostname()}, newPort) - _, ok := d.telemetryRetries.claim("peer-uuid", replacementTarget, time.Now()) - require.False(t, ok, "replacement endpoint started while old endpoint telemetry was in flight") - release(releaseOld) - waitFor(telemetryDone, "old endpoint result") - - _, ok = d.telemetryRetries.claim("peer-uuid", replacementTarget, time.Now()) - require.True(t, ok, "old endpoint failure backed off the replacement endpoint") -} - -// One peer holds its response open across several ticks. Other peers must keep -// receiving fresh telemetry instead of waiting for that slow request to finish. -func TestTelemetryLoop_KeepsHealthyNodeOnCadenceWhilePeerIsBlocked(t *testing.T) { - slowStarted := make(chan struct{}, 1) - slow := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - select { - case slowStarted <- struct{}{}: - default: - } - <-r.Context().Done() - })) - defer slow.Close() - slowURL, err := url.Parse(slow.URL) - require.NoError(t, err, "parse slow server URL") - slowPort, err := strconv.Atoi(slowURL.Port()) - require.NoError(t, err, "parse slow server port") - - healthyThird := make(chan struct{}, 1) - var healthyCalls atomic.Int32 - healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if healthyCalls.Add(1) >= 3 { - select { - case healthyThird <- struct{}{}: - default: - } - } - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-healthy"})) - })) - defer healthy.Close() - healthyURL, err := url.Parse(healthy.URL) - require.NoError(t, err, "parse healthy server URL") - healthyPort, err := strconv.Atoi(healthyURL.Port()) - require.NoError(t, err, "parse healthy server port") - - d := &daemon{ - dir: newDirectory(), - http: &http.Client{Timeout: time.Second}, - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-slow", - IP: slowURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: slowPort}, - }, - }) - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-healthy", - IP: healthyURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: healthyPort}, - }, - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan struct{}) - go func() { - d.runTelemetryLoop(ctx, 5*time.Millisecond) - close(done) - }() - - select { - case <-slowStarted: - case <-time.After(time.Second): - require.FailNow(t, "blocked peer telemetry did not start") - } - select { - case <-healthyThird: - case <-time.After(500 * time.Millisecond): - require.FailNow(t, "blocked peer delayed repeated healthy telemetry polls") - } - cancel() - select { - case <-done: - case <-time.After(time.Second): - require.FailNow(t, "telemetry loop did not drain blocked work after cancellation") - } -} - -// A tick can arrive while the preceding request is still running. The node must -// keep its in-flight reservation until completion to avoid duplicate polls. -func TestTelemetryLoop_DoesNotOverlapNodePolls(t *testing.T) { - var active atomic.Int32 - var maxActive atomic.Int32 - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - current := active.Add(1) - defer active.Add(-1) - calls.Add(1) - for { - seen := maxActive.Load() - if current <= seen || maxActive.CompareAndSwap(seen, current) { - break - } - } - select { - case <-time.After(20 * time.Millisecond): - assert.NoError(t, json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-a"})) - case <-r.Context().Done(): - } - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - - d := &daemon{ - dir: newDirectory(), - http: server.Client(), - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-a", - IP: serverURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, - }) - ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond) - defer cancel() - d.runTelemetryLoop(ctx, 5*time.Millisecond) - - require.GreaterOrEqual(t, calls.Load(), int32(2), "telemetry loop request count") - require.Equal(t, int32(1), maxActive.Load(), "maximum concurrent requests for one node") -} - -// Requests remain blocked across multiple ticks to expose a per-tick worker limit. -// The concurrency allowance must cover all outstanding work, including earlier ticks. -func TestTelemetryLoop_BoundsConcurrentWorkAcrossTicks(t *testing.T) { - const nodeCount = telemetryRefreshConcurrency + 1 - entered := make(chan struct{}, nodeCount) - releaseWork := make(chan struct{}) - var releaseOnce sync.Once - var active atomic.Int32 - var maxActive atomic.Int32 - d := &daemon{ - dir: newDirectory(), - http: &http.Client{Timeout: time.Second}, - } - - for i := range nodeCount { - nodeID := "node-" + strconv.Itoa(i) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - current := active.Add(1) - defer active.Add(-1) - for { - seen := maxActive.Load() - if current <= seen || maxActive.CompareAndSwap(seen, current) { - break - } - } - entered <- struct{}{} - <-releaseWork - w.WriteHeader(http.StatusServiceUnavailable) - })) - t.Cleanup(server.Close) - serverURL, err := url.Parse(server.URL) - require.NoError(t, err, "parse server URL") - port, err := strconv.Atoi(serverURL.Port()) - require.NoError(t, err, "parse server port") - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: nodeID, - IP: serverURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, - }) - } - t.Cleanup(func() { - releaseOnce.Do(func() { close(releaseWork) }) - }) - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - d.runTelemetryLoop(ctx, 5*time.Millisecond) - close(done) - }() - - for range telemetryRefreshConcurrency { - select { - case <-entered: - case <-time.After(2 * time.Second): - releaseOnce.Do(func() { close(releaseWork) }) - cancel() - <-done - require.FailNow(t, "telemetry loop did not fill its concurrency allowance") - } - } - overflowed := false - select { - case <-entered: - overflowed = true - case <-time.After(50 * time.Millisecond): - } - - releaseOnce.Do(func() { close(releaseWork) }) - if !overflowed { - select { - case <-entered: - case <-time.After(time.Second): - cancel() - <-done - require.FailNow(t, "queued telemetry did not start when capacity became available") - } - } - cancel() - select { - case <-done: - case <-time.After(time.Second): - require.FailNow(t, "telemetry loop did not drain bounded workers after cancellation") - } - - require.False(t, overflowed, "more than (%v)", telemetryRefreshConcurrency) - require.Equal(t, int32(telemetryRefreshConcurrency), maxActive.Load(), "maximum concurrent telemetry work") -} diff --git a/services/nvpair-node-scanner/tls.go b/services/nvpair-node-scanner/tls.go deleted file mode 100644 index cb9992b55..000000000 --- a/services/nvpair-node-scanner/tls.go +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "net/http" - "os" - "time" -) - -// tls.go is the node-scanner's OPTIONAL client-side TLS for enriching a node over -// its node-info (ni) port. It is dormant by default: with none of the -// --client-cert / --client-key / --ca-bundle flags set, buildTLSClient returns -// nil and the daemon fetches node-info over plain HTTP — the standard path, since -// node-info is plain per the transport policy. The scaffolding is kept so an -// operator can move node-info enrichment onto TLS in the future purely via these -// flags, with no code change and — deliberately — no per-node tls-port=/mtls= TXT -// self-report (that self-report was dropped to keep the transport policy static: -// a service owns its transport, the record carries only ports). - -// tlsClientOptions captures the three cert-related CLI knobs. All three are -// optional: no flags -> plain HTTP (nil client); --ca-bundle alone -> verify -// server certs against that bundle (server-only TLS); adding --client-cert + -// --client-key on top -> mTLS. -type tlsClientOptions struct { - CertPath string - KeyPath string - CABundlePath string -} - -// validate enforces the cert/key pairing rule: both are given, or neither. A bare -// --ca-bundle is always fine — it just means "trust these CAs but don't -// authenticate myself". -func (o tlsClientOptions) validate() error { - switch { - case o.CertPath != "" && o.KeyPath == "": - return errors.New("--client-cert requires --client-key") - case o.KeyPath != "" && o.CertPath == "": - return errors.New("--client-key requires --client-cert") - } - return nil -} - -// configured reports whether any TLS knob is set, i.e. whether the operator has -// opted node-info enrichment onto HTTPS. -func (o tlsClientOptions) configured() bool { - return o.CertPath != "" || o.KeyPath != "" || o.CABundlePath != "" -} - -// buildTLSClient produces an *http.Client for HTTPS node-info fetches. It keeps -// the system trust store as a base; --ca-bundle adds roots on top of that so an -// operator deploying a private CA doesn't also have to remove the OS roots. -// -// Returns nil when nothing is configured — the caller falls back to its -// plain-HTTP client so plain nodes keep working out of the box. -func buildTLSClient(o tlsClientOptions, timeout time.Duration) (*http.Client, error) { - if !o.configured() { - return nil, nil - } - cfg := &tls.Config{MinVersion: tls.VersionTLS12} - if o.CABundlePath != "" { - caPEM, err := os.ReadFile(o.CABundlePath) - if err != nil { - return nil, fmt.Errorf("read CA bundle: %w", err) - } - // Start from the system roots so we don't turn off WebPKI when the - // operator only wants to *add* a private CA. SystemCertPool can fail on - // some minimal Linux containers, in which case we fall back to an empty - // pool. - pool, err := x509.SystemCertPool() - if err != nil || pool == nil { - pool = x509.NewCertPool() - } - if !pool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("CA bundle %s contains no PEM certificates", o.CABundlePath) - } - cfg.RootCAs = pool - } - if o.CertPath != "" { - cert, err := tls.LoadX509KeyPair(o.CertPath, o.KeyPath) - if err != nil { - return nil, fmt.Errorf("load client cert/key: %w", err) - } - cfg.Certificates = []tls.Certificate{cert} - } - return &http.Client{ - Timeout: timeout, - Transport: nodeInfoTransport(cfg), - }, nil -} diff --git a/services/nvpair-node-scanner/tls_test.go b/services/nvpair-node-scanner/tls_test.go deleted file mode 100644 index 7da11a1dc..000000000 --- a/services/nvpair-node-scanner/tls_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "encoding/pem" - "net" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestTLSClientOptions_ValidatesConfiguration(t *testing.T) { - test := func(name string, options tlsClientOptions, wantOK bool) { - t.Run(name, func(t *testing.T) { - err := options.validate() - if wantOK { - require.NoError(t, err) - } else { - require.Error(t, err) - } - }) - } - test("none", tlsClientOptions{}, true) - test("ca only", tlsClientOptions{CABundlePath: "ca.pem"}, true) - test("cert+key", tlsClientOptions{CertPath: "c", KeyPath: "k"}, true) - test("cert without key", tlsClientOptions{CertPath: "c"}, false) - test("key without cert", tlsClientOptions{KeyPath: "k"}, false) -} - -// TestBuildTLSClientUnconfiguredIsNil is the dormant-by-default guarantee: with -// no flags set, there's no TLS client and the daemon falls back to plain HTTP. -func TestBuildTLSClientUnconfigured_IsNil(t *testing.T) { - c, err := buildTLSClient(tlsClientOptions{}, nodeInfoFetchTimeout) - require.NoError(t, err, "buildTLSClient(unconfigured) err") - require.Nil(t, c, "buildTLSClient(unconfigured) should return a nil client (plain-HTTP fallback)") -} - -func TestBuildTLSClient_AcceptsValidCAAndRejectsMalformedBundle(t *testing.T) { - srv := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) - defer srv.Close() - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) - dir := t.TempDir() - - good := filepath.Join(dir, "ca.pem") - require.NoError(t, os.WriteFile(good, caPEM, 0o600)) - c, err := buildTLSClient(tlsClientOptions{CABundlePath: good}, nodeInfoFetchTimeout) - require.NoError(t, err, "buildTLSClient(valid CA) (%v, %v)", c, err) - require.NotNil(t, c, "buildTLSClient(valid CA) (%v, %v)", c, err) - - bad := filepath.Join(dir, "bad.pem") - require.NoError(t, os.WriteFile(bad, []byte("not a pem"), 0o600)) - _, err = buildTLSClient(tlsClientOptions{CABundlePath: bad}, nodeInfoFetchTimeout) - require.Error(t, err, "buildTLSClient(garbage CA) should error") -} - -// TestFetchNodeInfoTLSClientSelection proves the flag-gated wiring: an HTTPS-only -// node-info endpoint is reachable only when the daemon holds a configured TLS -// client; the default plain-HTTP client cannot speak to it. -func TestFetchNodeInfo_SelectsClientForConfiguredTransport(t *testing.T) { - want := NodeInfoResponse{GPUs: []GPUInfo{{}}} - srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - w.WriteHeader(http.StatusNotFound) - return - } - assert.NoError(t, json.NewEncoder(w).Encode(want)) - })) - defer srv.Close() - - host, portStr, err := net.SplitHostPort(srv.Listener.Addr().String()) - require.NoError(t, err) - port, err := strconv.Atoi(portStr) - require.NoError(t, err) - - // With the TLS client (which trusts the test server), the HTTPS fetch works. - dTLS := &daemon{http: &http.Client{Timeout: nodeInfoFetchTimeout}, tlsHTTP: srv.Client()} - _, ok := dTLS.fetchNodeInfo(host, port) - assert.True(t, ok, "fetchNodeInfo with a TLS client should reach the HTTPS node-info endpoint") - - // Without it (dormant default), plain HTTP can't reach an HTTPS-only endpoint. - dPlain := &daemon{http: &http.Client{Timeout: nodeInfoFetchTimeout}} - _, ok = dPlain.fetchNodeInfo(host, port) - assert.False(t, ok, "fetchNodeInfo without a TLS client should not reach an HTTPS-only endpoint over plain HTTP") -} diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 5da30cbe0..78183e1bf 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -653,10 +653,10 @@ Attach to a pre-existing endpoint: TLS flags or a custom port. - **Manual-node persistence across restarts.** `nvpair-manual-nodes` remains an in-memory worker; the broker owns `configs/manual-nodes.json`, clears stale live projections when the worker exits, and replays the durable entries into its replacement. - **Per-event push semantics.** `discovery:nodes-changed` always carries the full current snapshot, not a delta. For small N this is fine and lets the client treat the payload as authoritative without state reconciliation. `errors:update` is likewise a full snapshot. -- **BYO-TLS node-info discovery.** The broker does not expose scanner's operator - TLS flags. Scanner's remote hardware reads currently use public HTTP, even - though node-info also offers peer node inventory over pinned mTLS. - Migrating those reads remains pending work in the [scanner spec](../nvpair-node-scanner/spec.md). +- **Hardware provenance.** Scanner reads verified remote inventory over exact-pin + mTLS and local inventory over private worker IPC. The + [scanner spec](../nvpair-node-scanner/spec.md) defines the ownership and + authentication requirements for hardware used by services. - **Per-worker log-level granularity.** `log/set-level` fans the broker's single level out to every running child, but there's no way to set a different level per worker — it's one level for the whole process tree. - **AuthN / AuthZ.** Transport security relies on the parent's pipe / socket ACL. There is no per-message token. - **HTTP / WebSocket transport.** JSON-RPC over stdio or pipe only — no web bridge yet. Likely lives in a separate component if/when added. diff --git a/services/shared/clustertrust/mesh.go b/services/shared/clustertrust/mesh.go index 664c4da7b..167fc0f42 100644 --- a/services/shared/clustertrust/mesh.go +++ b/services/shared/clustertrust/mesh.go @@ -61,10 +61,8 @@ func ClusterUUIDFromTXT(txt []string) string { // // Self-trust: a node always trusts its own leaf (it isn't pinned in trusted/, // which only holds peers). Without it a node could not reach its own gated -// endpoints — e.g. nvpair-node-scanner reading the local nvpair-node-info over the -// LAN, which it discovers via mDNS like any peer. With self-trust the local -// read completes over the same mTLS path as a peer read, so node-info needs no -// separate plaintext loopback listener. +// endpoints. Scanner's local node-info measurements instead arrive from its +// owned worker over private IPC, independently of membership or self-trust. // // A nil *Mesh reads as permanently unclustered so a service with no cluster dir // needs no special case. @@ -236,6 +234,13 @@ func (m *Mesh) HasPin(uuid string) bool { return ok } +// VerifyPeerCertificate checks a completed read against the current exact pin. +// Call Refresh before this check: a reused connection or an in-flight response +// can outlive the pin that authorized its handshake. +func (m *Mesh) VerifyPeerCertificate(uuid string, cert *x509.Certificate) bool { + return cert != nil && UUIDFromCert(cert) == uuid && m.matchDER(uuid, cert.Raw) +} + // matchDER is the server-side gate: accept der for uuid when it byte-for-byte // matches the pinned peer cert, or when uuid is this node and der is its own // leaf (self-trust). diff --git a/services/shared/clustertrust/mesh_test.go b/services/shared/clustertrust/mesh_test.go index 643f367bb..74cab6336 100644 --- a/services/shared/clustertrust/mesh_test.go +++ b/services/shared/clustertrust/mesh_test.go @@ -5,6 +5,7 @@ package clustertrust import ( "crypto/tls" + "crypto/x509" "net/http" "net/http/httptest" "os" @@ -23,6 +24,28 @@ func writeIdentity(t *testing.T, dir string, certPEM, keyPEM []byte) { require.NoError(t, os.WriteFile(filepath.Join(dir, "node.key"), keyPEM, 0o600)) } +func TestMesh_VerifyPeerCertificateUsesCurrentExactPin(t *testing.T) { + selfPEM, selfKey, _ := genLeaf(t, "self") + peerPEM, _, peerDER := genLeaf(t, "peer") + replacementPEM, _, replacementDER := genLeaf(t, "peer") + mesh := loadMeshDir(t, selfPEM, selfKey, map[string]string{"peer": string(peerPEM)}) + peer, err := x509.ParseCertificate(peerDER) + require.NoError(t, err) + replacement, err := x509.ParseCertificate(replacementDER) + require.NoError(t, err) + require.True(t, mesh.VerifyPeerCertificate("peer", peer)) + assert.False(t, mesh.VerifyPeerCertificate("another-peer", peer)) + assert.False(t, mesh.VerifyPeerCertificate("peer", replacement)) + assert.False(t, mesh.VerifyPeerCertificate("peer", nil)) + writePin(t, mesh.dir, "peer", string(replacementPEM)) + mesh.Refresh() + assert.False(t, mesh.VerifyPeerCertificate("peer", peer)) + require.True(t, mesh.VerifyPeerCertificate("peer", replacement)) + require.NoError(t, os.Remove(filepath.Join(mesh.dir, "trusted", "peer.json"))) + mesh.Refresh() + assert.False(t, mesh.VerifyPeerCertificate("peer", replacement)) +} + func loadMeshDir(t *testing.T, certPEM, keyPEM []byte, pins map[string]string) *Mesh { t.Helper() dir := t.TempDir() diff --git a/services/shared/clustertrust/peerclient.go b/services/shared/clustertrust/peerclient.go index f6fa7b026..ebcb257f0 100644 --- a/services/shared/clustertrust/peerclient.go +++ b/services/shared/clustertrust/peerclient.go @@ -35,9 +35,6 @@ const ( // instead: a warm POST to these endpoints is single-digit milliseconds, so // hundreds of queued frames drain well inside the per-request timeout. peerMaxConnsPerHost = 8 - // peerMaxIdleConnsPerHost keeps the whole cap warm between bursts, so a - // steady trickle of events never re-handshakes. - peerMaxIdleConnsPerHost = peerMaxConnsPerHost // peerMaxIdleConns bounds the pool across every peer in a cluster of this // scale (~a dozen nodes). peerMaxIdleConns = 64 @@ -73,6 +70,10 @@ const ( type PeerClientOptions struct { Timeout time.Duration ResponseHeaderTimeout time.Duration + // DialTimeout and MaxConnsPerHost override the defaults for tighter probes. + // Non-positive values retain the pool defaults. + DialTimeout time.Duration + MaxConnsPerHost int // ResolvePin overrides the Mesh's disk-backed peer-pin lookup. The resolver // is the authorization source and is called without the pool lock held. ResolvePin func(peerUUID string) ([]byte, bool) @@ -162,7 +163,13 @@ func (p *PeerClientPool) Client(peerUUID string) (*http.Client, bool) { pinnedDER: bytes.Clone(der), selfDER: bytes.Clone(self), } - entry.client = &http.Client{Timeout: p.opts.Timeout, Transport: entry.transport} + // A redirect must not move an authenticated request onto another transport + // or endpoint. Cluster service endpoints do not require redirection. + entry.client = &http.Client{ + Timeout: p.opts.Timeout, + Transport: entry.transport, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } p.entries[peerUUID] = entry return entry.client, true } @@ -188,6 +195,13 @@ func (p *PeerClientPool) newTransport(tlsCfg *tls.Config) *http.Transport { if dialTimeout <= 0 { dialTimeout = 30 * time.Second } + if p.opts.DialTimeout > 0 { + dialTimeout = p.opts.DialTimeout + } + maxConns := peerMaxConnsPerHost + if p.opts.MaxConnsPerHost > 0 { + maxConns = p.opts.MaxConnsPerHost + } return &http.Transport{ DialContext: (&net.Dialer{ Timeout: dialTimeout, @@ -196,9 +210,9 @@ func (p *PeerClientPool) newTransport(tlsCfg *tls.Config) *http.Transport { TLSClientConfig: tlsCfg, TLSHandshakeTimeout: dialTimeout, ResponseHeaderTimeout: p.opts.ResponseHeaderTimeout, - MaxConnsPerHost: peerMaxConnsPerHost, + MaxConnsPerHost: maxConns, MaxIdleConns: peerMaxIdleConns, - MaxIdleConnsPerHost: peerMaxIdleConnsPerHost, + MaxIdleConnsPerHost: maxConns, IdleConnTimeout: PeerIdleTimeout, } } diff --git a/services/shared/clustertrust/peerclient_test.go b/services/shared/clustertrust/peerclient_test.go index fc2be6f08..cccfc5531 100644 --- a/services/shared/clustertrust/peerclient_test.go +++ b/services/shared/clustertrust/peerclient_test.go @@ -51,6 +51,15 @@ func checkPeerClientCertificates(t *testing.T, client *http.Client, selfDER, pee } } +func TestPeerClientPool_NodeInfoOptionsPreserveTransportBounds(t *testing.T) { + pool := NewPeerClientPoolOpts(Open(""), PeerClientOptions{Timeout: 3 * time.Second, DialTimeout: 250 * time.Millisecond, MaxConnsPerHost: 1}) + transport := pool.newTransport(nil) + assert.Equal(t, 250*time.Millisecond, transport.TLSHandshakeTimeout) + assert.Equal(t, 1, transport.MaxConnsPerHost) + assert.Equal(t, 1, transport.MaxIdleConnsPerHost) + assert.Equal(t, PeerIdleTimeout, transport.IdleConnTimeout) +} + // TestPeerClientPool_ReusesOneConnection is the regression for the dropped // inter-node event bug: building a client per request meant every event paid a // fresh mTLS handshake and leaked the socket afterwards, so a burst exhausted @@ -122,7 +131,7 @@ func TestPeerClientPool_BoundsIdleConnections(t *testing.T) { require.True(t, ok, "pinned peer must yield a client") entry := pool.entries["uuid-peer"] assert.Equal(t, PeerIdleTimeout, entry.transport.IdleConnTimeout) - assert.Equal(t, peerMaxIdleConnsPerHost, entry.transport.MaxIdleConnsPerHost) + assert.Equal(t, peerMaxConnsPerHost, entry.transport.MaxIdleConnsPerHost) // The bound that actually protects the peer. Retaining idle connections stops // the leak, but only capping TOTAL connections per host stops a per-event // fan-out from opening hundreds of simultaneous handshakes during a burst — diff --git a/services/shared/discovery/discovery.go b/services/shared/discovery/discovery.go index b89399186..9559f5ec9 100644 --- a/services/shared/discovery/discovery.go +++ b/services/shared/discovery/discovery.go @@ -51,11 +51,14 @@ import ( "github.com/miekg/dns" ) +// EventType identifies a change to the discovered set. +type EventType string + // Event types emitted by Run. const ( - Discovered = "discovered" - Updated = "updated" - Removed = "removed" + Discovered EventType = "discovered" + Updated EventType = "updated" + Removed EventType = "removed" ) // probeConcurrency bounds how many liveness probes run at once. A probe against @@ -147,7 +150,7 @@ type Node struct { // Event is a change to the discovered set. type Event struct { - Type string // Discovered, Updated, or Removed + Type EventType Node Node } diff --git a/services/shared/discovery/discovery_test.go b/services/shared/discovery/discovery_test.go index 738469178..dd623130e 100644 --- a/services/shared/discovery/discovery_test.go +++ b/services/shared/discovery/discovery_test.go @@ -29,8 +29,8 @@ func seenSet(ns ...Node) map[string]Node { } // eventsByType tallies events for concise assertions. -func eventsByType(evs []Event) map[string]int { - m := map[string]int{} +func eventsByType(evs []Event) map[EventType]int { + m := map[EventType]int{} for _, e := range evs { m[e.Type]++ } @@ -337,7 +337,7 @@ func TestRun_EmitsAndCloses(t *testing.T) { done := make(chan struct{}) go func() { b.Run(ctx, events); close(done) }() - got := map[string]int{} + got := map[EventType]int{} timeout := time.After(2 * time.Second) for { select { diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index b3f6601fb..cb80450fd 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -30,6 +30,8 @@ package noderec import ( "fmt" + "maps" + "slices" "sort" "strconv" "strings" @@ -115,8 +117,9 @@ var serviceKeyOrder = []ServiceKey{ type Transport int const ( - // TransportPlain: always plain HTTP, even on a clustered node (node-info and - // the local inference engines). + // TransportPlain describes public node-info hardware displays and local + // inference engines. Services read remote node-info separately over pinned + // mTLS; this public transport policy does not authorize service measurements. TransportPlain Transport = iota // TransportMTLSWhenClustered: cluster-scoped pin-based mTLS when the target // node is clustered, else plain (nvpair-errors, nvpair-workload-manager). @@ -140,8 +143,8 @@ func (s ServiceKey) Transport() Transport { } // UsesMTLS reports whether a consumer should dial this service over cluster -// mTLS, given whether the target node is clustered. node-info is deliberately -// plain even on a clustered node; errors/workload gate on the target's cluster +// mTLS, given whether the target node is clustered. Public node-info displays +// stay plain even on a clustered node; errors/workload gate on the target's cluster // state; the cluster-manager split channel's trusted plane is mTLS. func (s ServiceKey) UsesMTLS(targetClustered bool) bool { switch s.Transport() { @@ -302,30 +305,16 @@ const ( // converges on the cluster principal even though the scanner spawned (and // minted its own node-id) before cluster-manager wrote identity.json. MethodReloadIdentity = "discovery:reload-identity" - // MethodReloadTrust tells the daemon that this node's cluster pin set changed, - // so it must re-derive the trusted annotation on every directory entry. The - // daemon otherwise answers "do I hold a pin for this peer?" only when that - // peer's mDNS record moves, and a peer already advertising when we join never - // moves again — leaving it annotated from before our pins existed. The broker - // triggers this off nvpair-cluster-manager, which announces only after the pin - // is on disk, so the daemon always reads a directory at least as new as the - // event. + // MethodReloadTrust invalidates peer proofs whose exact pins changed and + // schedules authentication of pending discoveries. The broker sends it + // after cluster-manager writes pins; an announcement alone cannot grant trust. MethodReloadTrust = "discovery:reload-trust" // MethodSetClusterIdentity tells nvpair-node-info the cluster principal this // node currently holds, so it can report it on /v1/node-info. // - // node-info is spawned with no cluster dir on purpose — it is the one - // inter-node surface kept plain so any LAN peer can read this host's - // inventory (see the broker's spawnNodeInfo) — which also leaves it unable to - // read membership for itself. The broker owns that fact and pushes it here on - // startup and on every change, so node-info reports one live value rather - // than deriving a second one. - // - // The value is a peer's only mDNS-independent way to learn that this node has - // left a cluster: cluster membership otherwise reaches the fleet solely as the - // cluster-uuid= TXT key, and a consumer that stops receiving this node's mDNS - // record keeps its last observed value indefinitely. + // With --cluster-dir, node-info reads membership from the mesh. Standalone + // deployments without a directory receive the identity from their parent. MethodSetClusterIdentity = "nodeinfo:set-cluster-identity" MethodSetPeerMetadata = "nodeinfo:set-peer-metadata" NotifyInfoUpdated = "nodeinfo:info-updated" @@ -587,7 +576,7 @@ type DirectoryNode struct { // only answer. Empty for a node that published one address. IPs []string `json:"ips,omitempty"` ClusterUUID string `json:"clusterUuid,omitempty"` - Trusted bool `json:"trusted"` // we hold a cluster pin for it + Trusted bool `json:"trusted"` // verified peer with a current pin, or local membership Services map[ServiceKey]ServiceStatus `json:"services"` GPUs []GPUInfo `json:"gpus,omitempty"` CPU *CPUInfo `json:"cpu,omitempty"` @@ -614,6 +603,42 @@ type DirectoryNode struct { LastSeen int64 `json:"lastSeen"` // Unix seconds } +// Equals compares all fields by value. Slice order matters, and nil collections +// differ from empty collections. +func (n DirectoryNode) Equals(other DirectoryNode) bool { + return n.DiscoveryRecordID == other.DiscoveryRecordID && + n.IdentityVerification == other.IdentityVerification && + n.HostUUID == other.HostUUID && + n.Name == other.Name && + n.IP == other.IP && + equalSlice(n.IPs, other.IPs) && + n.ClusterUUID == other.ClusterUUID && + n.Trusted == other.Trusted && + (n.Services == nil) == (other.Services == nil) && maps.Equal(n.Services, other.Services) && + equalSlice(n.GPUs, other.GPUs) && + equalOptional(n.CPU, other.CPU) && + equalOptional(n.Memory, other.Memory) && + equalSlice(n.Models, other.Models) && + (n.ModelsByEngine == nil) == (other.ModelsByEngine == nil) && + maps.EqualFunc(n.ModelsByEngine, other.ModelsByEngine, equalSlice[string]) && + (n.LoadedByEngine == nil) == (other.LoadedByEngine == nil) && + maps.EqualFunc(n.LoadedByEngine, other.LoadedByEngine, equalSlice[string]) && + n.LastSeen == other.LastSeen +} + +// equalSlice compares ordered values while distinguishing nil from empty slices. +func equalSlice[T comparable](a, b []T) bool { + return (a == nil) == (b == nil) && slices.Equal(a, b) +} + +// equalOptional distinguishes an absent value from a present zero value. +func equalOptional[T comparable](a, b *T) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} + // Clustered reports whether the node advertises a cluster identity. func (n DirectoryNode) Clustered() bool { return n.ClusterUUID != "" } diff --git a/services/shared/noderec/noderec_test.go b/services/shared/noderec/noderec_test.go index e173d177b..4479151a9 100644 --- a/services/shared/noderec/noderec_test.go +++ b/services/shared/noderec/noderec_test.go @@ -198,6 +198,64 @@ func TestDirectoryNode_ReportsMembershipAndServicePresence(t *testing.T) { assert.False(t, (DirectoryNode{}).Clustered(), "empty node should not be Clustered") } +func TestDirectoryNode_EqualsComparesValues(t *testing.T) { + original := DirectoryNode{ + DiscoveryRecordID: "peer:host-1", IdentityVerification: IdentityPinned, + HostUUID: "host-1", ClusterUUID: "host-1", Trusted: true, + Name: "host", IP: "192.0.2.20", IPs: []string{"192.0.2.20"}, + Services: map[ServiceKey]ServiceStatus{ServiceNodeInfo: {Port: 14318}}, + GPUs: []GPUInfo{{Name: "GPU", VramBytes: 1024}}, + CPU: &CPUInfo{Name: "CPU", Cores: 8}, + Memory: &MemoryInfo{TotalBytes: 4096}, + Models: []string{"model-a"}, + ModelsByEngine: map[string][]string{"ollama": {"model-a"}}, + LoadedByEngine: map[string][]string{"ollama": {"model-a"}}, + LastSeen: 123, + } + // Decoding produces separate allocations with the same values. + body, err := json.Marshal(original) + require.NoError(t, err) + var decoded DirectoryNode + require.NoError(t, json.Unmarshal(body, &decoded)) + assert.True(t, original.Equals(decoded)) + assert.True(t, decoded.Equals(original)) + assert.True(t, (DirectoryNode{}).Equals(DirectoryNode{})) +} + +func TestDirectoryNode_EqualsDistinguishesRecordChanges(t *testing.T) { + test := func(name string, a, b DirectoryNode) { + t.Run(name, func(t *testing.T) { + assert.False(t, a.Equals(b)) + assert.False(t, b.Equals(a)) + }) + } + test("verification changes", DirectoryNode{IdentityVerification: IdentityUnverified}, DirectoryNode{IdentityVerification: IdentityPinned}) + test("address order changes", DirectoryNode{IPs: []string{"192.0.2.20", "192.0.2.21"}}, DirectoryNode{IPs: []string{"192.0.2.21", "192.0.2.20"}}) + test("model list changes", DirectoryNode{Models: []string{"model-a"}}, DirectoryNode{Models: []string{"model-b"}}) + test("model attribution changes", DirectoryNode{ModelsByEngine: map[string][]string{"ollama": {"model-a"}}}, DirectoryNode{ModelsByEngine: map[string][]string{"lmstudio": {"model-a"}}}) + test("loaded models change", DirectoryNode{LoadedByEngine: map[string][]string{"ollama": {"model-a"}}}, DirectoryNode{LoadedByEngine: map[string][]string{"ollama": {"model-b"}}}) + test("timestamp changes", DirectoryNode{LastSeen: 1}, DirectoryNode{LastSeen: 2}) +} + +func TestDirectoryNode_EqualsDistinguishesAbsentAndEmptyValues(t *testing.T) { + test := func(name string, a, b DirectoryNode) { + t.Run(name, func(t *testing.T) { + assert.False(t, a.Equals(b)) + assert.False(t, b.Equals(a)) + }) + } + test("addresses", DirectoryNode{}, DirectoryNode{IPs: []string{}}) + test("services", DirectoryNode{}, DirectoryNode{Services: map[ServiceKey]ServiceStatus{}}) + test("GPUs", DirectoryNode{}, DirectoryNode{GPUs: []GPUInfo{}}) + test("CPU", DirectoryNode{}, DirectoryNode{CPU: &CPUInfo{}}) + test("memory", DirectoryNode{}, DirectoryNode{Memory: &MemoryInfo{}}) + test("models", DirectoryNode{}, DirectoryNode{Models: []string{}}) + test("models by engine", DirectoryNode{}, DirectoryNode{ModelsByEngine: map[string][]string{}}) + test("loaded models by engine", DirectoryNode{}, DirectoryNode{LoadedByEngine: map[string][]string{}}) + test("engine model list", DirectoryNode{ModelsByEngine: map[string][]string{"ollama": nil}}, DirectoryNode{ModelsByEngine: map[string][]string{"ollama": {}}}) + test("engine loaded model list", DirectoryNode{LoadedByEngine: map[string][]string{"ollama": nil}}, DirectoryNode{LoadedByEngine: map[string][]string{"ollama": {}}}) +} + func TestDirectoryNode_RoundTripsJSON(t *testing.T) { orig := DirectoryNode{ HostUUID: "host-1", Name: "host", IP: "192.168.1.10", ClusterUUID: "clu", diff --git a/services/tests/models_http_test.go b/services/tests/models_http_test.go index 27f094c98..9f4fb71ad 100644 --- a/services/tests/models_http_test.go +++ b/services/tests/models_http_test.go @@ -1,13 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Cross-process test for the models-http path: a node advertises an -// engine-manager (em) service on its _nvpair-node record, and the promoted -// discovery daemon enriches the node's model list by fetching /v1/models over -// plain HTTP from that em port — replacing the retired models= TXT key. We stand -// in for engine-manager with a tiny HTTP server, advertise em= + ip so the -// daemon dials it over loopback, and assert the enriched list surfaces on the -// broker's discovery:get-nodes response. +// Cross-process test for model inventory from a pinned cluster peer. Scanner +// authenticates its node-info metadata, then fetches /v1/models over mTLS and +// carries the flat, per-engine, and loaded inventories through the broker. package tests import ( @@ -24,15 +20,14 @@ import ( "github.com/stretchr/testify/require" "github.com/grandcat/zeroconf" + + "nvpair-shared/noderec" ) func TestDiscovery_EnrichesModelsThroughHTTP(t *testing.T) { - // Stand-in engine-manager HTTP surface serving the model list. - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/models" { - http.NotFound(w, r) - return - } + const instance = "xproc-models-1" + cluster := newInterNodeCluster(t) + cluster.startModelsPeer(t, instance, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Flat union + per-engine attribution, exactly the shape engine-manager's // ModelsResult serializes. The daemon must enrich both onto the node. assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ @@ -49,62 +44,54 @@ func TestDiscovery_EnrichesModelsThroughHTTP(t *testing.T) { }, })) })) - t.Cleanup(srv.Close) - u, err := url.Parse(srv.URL) - require.NoError(t, err, "parse stub url") - emPort, err := strconv.Atoi(u.Port()) - require.NoError(t, err, "stub port") - const instance = "xproc-models-1" - // Advertise a _nvpair-node with em= and ip=127.0.0.1 so the daemon - // fetches the model list from the stub over loopback (ip= wins over the mDNS - // address in the daemon's enrichment target selection). - txt := []string{"v=1", "uuid=" + instance + "-uuid", "ip=127.0.0.1", fmt.Sprintf("em=%d", emPort)} - zsrv, err := zeroconf.Register(instance, nodeRecordService, testDomain, emPort, txt, nil) - require.NoError(t, err, "register (%v)", instance) - t.Cleanup(zsrv.Shutdown) - t.Logf("advertising %s @ %s (em=%d)", instance, nodeRecordService, emPort) - - stdin, msgs, cleanup := startBroker(t) + stdin, msgs, _, cleanup := startBrokerWithDirs(t, t.TempDir(), cluster.nodeDir) t.Cleanup(cleanup) - // Poll get-nodes until the node appears carrying the enriched model list. - deadline := time.After(30 * time.Second) - ticker := time.NewTicker(1500 * time.Millisecond) - defer ticker.Stop() - id := 100 - sendReq(t, stdin, id, "discovery:get-nodes") - for { - select { - case msg, ok := <-msgs: - require.True(t, ok, "broker stream closed unexpectedly") - if msg.Method == "" && msg.ID != nil { - var res availableNodesResult - require.NoError(t, json.Unmarshal(msg.Result, &res)) - - if n, found := findNode(res.Nodes, instance); found && - modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && - loadedByEngineMatch(n.LoadedByEngine) { - return // enriched flat + per-engine + loaded lists surfaced - } - - } - case <-ticker.C: - id++ - sendReq(t, stdin, id, "discovery:get-nodes") - case <-deadline: - require.FailNow(t, fmt.Sprintf("timed out waiting for %q with an enriched model list", instance)) - } - } + pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 30*time.Second, func(n availableNode) bool { + return modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && + loadedByEngineMatch(n.LoadedByEngine) + }) } -func findNode(nodes []availableNode, id string) (availableNode, bool) { - for _, n := range nodes { - if n.ID == id { - return n, true +// startModelsPeer serves authenticated node metadata and a model handler using +// the same pinned identity. The one announcement remains fixed during refresh tests. +func (c *interNodeCluster) startModelsPeer(t *testing.T, instance string, models http.Handler) { + t.Helper() + var port int + mux := http.NewServeMux() + mux.HandleFunc("/v1/node-info", func(w http.ResponseWriter, r *http.Request) { + info := noderec.NodeInfo{ + HostUUID: c.peerUUID, ClusterUUID: &c.peerUUID, + Peer: &noderec.PeerMetadata{ + Name: instance, Addresses: []string{"127.0.0.1"}, + Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port, noderec.ServiceEngineManager: port}, + }, } - } - return availableNode{}, false + assert.NoError(t, json.NewEncoder(w).Encode(info)) + }) + mux.Handle("/v1/models", models) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := c.peer.VerifyClientPin(r); !ok { + http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) + return + } + mux.ServeHTTP(w, r) + })) + server.TLS = c.peer.ServerTLSConfig() + server.StartTLS() + t.Cleanup(server.Close) + u, err := url.Parse(server.URL) + require.NoError(t, err, "parse model peer URL") + port, err = strconv.Atoi(u.Port()) + require.NoError(t, err, "model peer port") + + txt := []string{"v=1", "uuid=" + c.peerUUID, "cluster-uuid=" + c.peerUUID, "ip=127.0.0.1", + fmt.Sprintf("ni=%d", port), fmt.Sprintf("em=%d", port)} + advertisement, err := zeroconf.Register(instance, nodeRecordService, testDomain, port, txt, nil) + require.NoError(t, err, "register model peer") + t.Cleanup(advertisement.Shutdown) + t.Logf("advertising %s with pinned node-info and models (port=%d)", instance, port) } func modelsMatch(models []string) bool { diff --git a/services/tests/models_refresh_test.go b/services/tests/models_refresh_test.go index 06d307cc2..f151c3fed 100644 --- a/services/tests/models_refresh_test.go +++ b/services/tests/models_refresh_test.go @@ -15,9 +15,6 @@ import ( "fmt" "io" "net/http" - "net/http/httptest" - "net/url" - "strconv" "sync" "testing" "time" @@ -25,9 +22,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grandcat/zeroconf" - "nvpair-shared/jsonrpc" + "nvpair-shared/noderec" ) // swapModels is a /v1/models stub whose body can be swapped at runtime, standing @@ -61,29 +57,25 @@ func (s *swapModels) handler(t *testing.T) http.HandlerFunc { func TestModelsPeriodicRefresh_ConvergesWithoutMDNSChange(t *testing.T) { // Start with an empty inventory (engine present but serving no models yet). stub := &swapModels{} - stub.set(map[string]any{"models": []string{}}) - srv := httptest.NewServer(stub.handler(t)) - t.Cleanup(srv.Close) - u, err := url.Parse(srv.URL) - require.NoError(t, err, "parse stub url") - emPort, err := strconv.Atoi(u.Port()) - require.NoError(t, err, "stub port") + emptyByEngine := map[string][]string{"ollama": {}, "lmstudio": {}, "llamacpp": {}} + stub.set(map[string]any{ + "models": []string{}, + "modelsByEngine": emptyByEngine, + "loadedByEngine": emptyByEngine, + }) const instance = "xproc-models-refresh-1" - // Advertise em= + ip=127.0.0.1 once and never touch the record again, - // so the only thing that changes during the test is the /v1/models body. - txt := []string{"v=1", "uuid=" + instance + "-uuid", "ip=127.0.0.1", fmt.Sprintf("em=%d", emPort)} - zsrv, err := zeroconf.Register(instance, nodeRecordService, testDomain, emPort, txt, nil) - require.NoError(t, err, "register (%v)", instance) - t.Cleanup(zsrv.Shutdown) - t.Logf("advertising %s @ %s (em=%d), models initially empty", instance, nodeRecordService, emPort) + cluster := newInterNodeCluster(t) + cluster.startModelsPeer(t, instance, stub.handler(t)) - stdin, msgs, cleanup := startBroker(t) + stdin, msgs, _, cleanup := startBrokerWithDirs(t, t.TempDir(), cluster.nodeDir) t.Cleanup(cleanup) - // Phase 1: the node is discovered and enriched, but with no models. - pollForNode(t, stdin, msgs, instance, 30*time.Second, func(n availableNode) bool { - return len(n.Models) == 0 + // Require a successful empty inventory read, not just an unverified row + // whose model fields have never been populated. + pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 30*time.Second, func(n availableNode) bool { + return len(n.Models) == 0 && byEngineEqual(n.ModelsByEngine, emptyByEngine) && + byEngineEqual(n.LoadedByEngine, emptyByEngine) }) t.Log("node discovered with an empty inventory; now serving models without an mDNS change") @@ -103,7 +95,7 @@ func TestModelsPeriodicRefresh_ConvergesWithoutMDNSChange(t *testing.T) { "llamacpp": {"org/router-model-GGUF:Q4_K_M"}, }, }) - pollForNode(t, stdin, msgs, instance, 45*time.Second, func(n availableNode) bool { + pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 45*time.Second, func(n availableNode) bool { return modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && loadedByEngineMatch(n.LoadedByEngine) }) @@ -124,16 +116,15 @@ func TestModelsPeriodicRefresh_ConvergesWithoutMDNSChange(t *testing.T) { "llamacpp": {}, }, }) - emptyByEngine := map[string][]string{"ollama": {}, "lmstudio": {}, "llamacpp": {}} - pollForNode(t, stdin, msgs, instance, 45*time.Second, func(n availableNode) bool { + pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 45*time.Second, func(n availableNode) bool { return len(n.Models) == 0 && byEngineEqual(n.ModelsByEngine, emptyByEngine) && byEngineEqual(n.LoadedByEngine, emptyByEngine) }) } -// pollForNode drives discovery:get-nodes on a cadence until the node named -// instance appears and satisfies pred, or the timeout elapses. -func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, instance string, timeout time.Duration, pred func(availableNode) bool) { +// pollForNode waits for the named, authenticated peer to satisfy pred. Matching +// its pinned identity prevents an unverified discovery from satisfying readiness. +func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, instance, hostUUID string, timeout time.Duration, pred func(availableNode) bool) { t.Helper() deadline := time.After(timeout) ticker := time.NewTicker(1500 * time.Millisecond) @@ -145,11 +136,23 @@ func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, ins case msg, ok := <-msgs: require.True(t, ok, "broker stream closed unexpectedly") if msg.Method == "" && msg.ID != nil { - var res availableNodesResult - require.NoError(t, json.Unmarshal(msg.Result, &res)) - - if n, found := findNode(res.Nodes, instance); found && pred(n) { - return + require.Nil(t, msg.Error, "discovery:get-nodes") + var res struct { + Nodes []struct { + availableNode + HostUUID string `json:"hostUuid"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + Trusted bool `json:"trusted"` + } `json:"nodes"` + } + require.NoError(t, json.Unmarshal(msg.Result, &res), "discovery:get-nodes") + for _, node := range res.Nodes { + if node.ID == instance && node.HostUUID == hostUUID && + node.IdentityVerification == noderec.IdentityPinned && node.Trusted && + node.DiscoveryRecordID != "" && pred(node.availableNode) { + return + } } } From f9f538d4114b763763b1dc93cf5646d44b53967b Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Tue, 20 Oct 2026 16:12:51 -0400 Subject: [PATCH 5/8] Preserve verified provenance through discovery consumers Keep unverified discoveries and manual configurations separate from verified host identities throughout broker storage, relay snapshots, service targets, and desktop/TUI rows. Candidate removal cannot clear a verified peer's scheduling sample or fail its workloads, and discovery claims never enter the scheduler's node universe. Publish and validate full discovery:node-info snapshots, retaining record provenance and elapsed sample age. Manual service measurements require current exact cluster pins and matching response identities. Preserve public frontend hardware previews and independently configured manual engine routes. Distinguish broker node-key removal from exact manual entry IDs so overlapping names cannot select another configuration. Preserve live sample age while it waits behind the scheduler feed lock, using the in-memory projection time and the existing telemetry aging helper. TestLiveTelemetry_IncludesTimeWaitingForSchedulerFeed covers a sample delayed behind replay without changing IPC payloads. Use explicit fixed time values in test fixtures. Extend the existing UUID-keying test with provenance and proxy-port assertions rather than duplicating its merge scenario in the identity-contract tests. Suggested review order: 1. docs/architecture.mdx, services/nvpair-node-scanner/spec.md, services/nvpair-manual-nodes/spec.md, desktop/docs/frontend-api.md, then services/shared/noderec/noderec.go: identity boundaries and hardware snapshot contracts. 2. services/nvpair-ui-broker/discovery_test.go and relay/relay_test.go, then discovery.go and relay/relay.go: independent discovery record keys, snapshot cache binding, and candidate removal preserving peer telemetry and workloads. 3. services/nvpair-node-scanner/daemon_test.go, then daemon.go: authenticated snapshot publication, notification queue age, and complete local removals. 4. services/nvpair-manual-nodes/manager_test.go, then manager.go and main.go; services/nvpair-ui-broker/manualnodes_test.go and manualnodes_persistence_test.go, then manualnodes.go and broker.go's relayToManualNodes: pinned probes, status equality, opaque manual IDs, and unambiguous removal selectors. 5. services/nvpair-ui-broker/broker_test.go, scheduler_test.go, and telemetry_test.go, then broker.go and telemetry.go: scheduler filtering, full snapshot forwarding, and live/replayed sample age, including time waiting for schedulerFeedMu. 6. services/nvpair-proxy/proxy_test.go and subscribed_test.go, then proxy.go; services/nvpair-engine-manager/remotepeers_test.go and remote_test.go, then remotepeers.go; services/nvpair-errors/manager_test.go and rename_test.go, then manager.go; services/nvpair-workload-manager/relaysource_test.go, then relaysource.go; services/nvpair-cluster-manager/mdns_browser_test.go, then mdns_browser.go: service-peer acceptance and rejection, plus selected discovery resolution. 7. desktop/tests/modular/node-identity-contract.test.ts, node-info-poll-cycle.test.ts, uuid-node-keying.test.ts, and empty-handlers.test.ts, then shared/types/cluster.ts, modular-state.ts, node-info-poller.ts, and empty-handlers.ts: distinct row keys, independent snapshot guards, same-row proxy merges, display-only HTTP hardware, and broker-key removal. useInvitablePeers.ts documents member matching with independent candidate rows. 8. services/nvpair-tui/ui/nodesmodel_test.go and nodetelemetry_test.go, then nodeswire.go, nodesmodel.go, and nodetelemetry.go: row identity, public previews, and candidate/peer removal. 9. services/tests/broker_management_test.go, cluster_data_plane_test.go, remote_engine_test.go, secure_inference_test.go, workload_identity_interop_test.go, and workload_interop_test.go: real-worker fixtures wait for the intended authenticated peer and preserve independent manual engine routes. Finish with affected READMEs and desktop/docs/services-parity.md. Validation for this amendment: 48 TypeScript test files pass, 281 tests pass and two existing platform cases remain skipped. All twelve Go service binaries and the shared module compile; no Go tests were run for this amendment. Node/web/test type checks, targeted ESLint, formatting, whitespace, and SPDX checks pass. Earlier validation observed failing regressions before provenance, removal, scheduler, local ownership, unchanged-sample, and queue-age fixes. Applicable shared and service Go suites and five controlled cross-process scheduler/node-info cases passed. The unchanged engine-manager Windows chmod fixture was excluded from its broad run because Windows still permits unlinking its fixture. Those Go suites were not rerun for this amendment. Signed-off-by: Kaylee Lubick --- desktop/docs/frontend-api.md | 11 +- desktop/docs/services-parity.md | 41 +- .../electron/service-bridge/empty-handlers.ts | 8 +- .../electron/service-bridge/modular-state.ts | 150 +++--- .../service-bridge/node-info-poller.ts | 24 +- .../src/shared/constants/modular-runtime.ts | 16 - desktop/src/shared/types/cluster.ts | 12 +- desktop/src/ui/hooks/useInvitablePeers.ts | 6 +- desktop/tests/modular/empty-handlers.test.ts | 51 +++ .../tests/modular/ghost-node-dedupe.test.ts | 275 ----------- .../modular/node-identity-contract.test.ts | 254 ++++++++++- .../modular/node-info-poll-cycle.test.ts | 86 ++++ .../tests/modular/uuid-node-keying.test.ts | 55 ++- docs/architecture.mdx | 9 +- .../nvpair-cluster-manager/mdns_browser.go | 62 ++- .../mdns_browser_test.go | 81 ++++ services/nvpair-engine-manager/remote_test.go | 7 +- services/nvpair-engine-manager/remotepeers.go | 23 +- .../nvpair-engine-manager/remotepeers_test.go | 142 +++++- services/nvpair-errors/manager.go | 9 +- services/nvpair-errors/manager_test.go | 57 +++ services/nvpair-errors/rename_test.go | 24 +- services/nvpair-manual-nodes/README.md | 22 +- services/nvpair-manual-nodes/main.go | 25 +- services/nvpair-manual-nodes/manager.go | 351 +++++++------- services/nvpair-manual-nodes/manager_test.go | 431 ++++++++++++++---- services/nvpair-manual-nodes/spec.md | 2 +- services/nvpair-manual-nodes/tls.go | 82 ---- services/nvpair-node-scanner/daemon.go | 22 +- services/nvpair-node-scanner/daemon_test.go | 108 ++++- .../nvpair-node-scanner/telemetry_test.go | 5 + services/nvpair-proxy/proxy.go | 19 +- services/nvpair-proxy/proxy_test.go | 70 +++ services/nvpair-proxy/subscribed_test.go | 68 ++- services/nvpair-tui/ui/nodes.go | 25 +- services/nvpair-tui/ui/nodesmodel.go | 72 +-- services/nvpair-tui/ui/nodesmodel_test.go | 180 ++++++-- services/nvpair-tui/ui/nodeswire.go | 7 +- services/nvpair-tui/ui/nodetelemetry.go | 10 +- services/nvpair-tui/ui/nodetelemetry_test.go | 27 +- services/nvpair-tui/ui/toast_test.go | 69 +++ services/nvpair-ui-broker/README.md | 15 +- services/nvpair-ui-broker/broker.go | 57 ++- services/nvpair-ui-broker/broker_test.go | 66 +++ services/nvpair-ui-broker/discovery.go | 153 ++++++- services/nvpair-ui-broker/discovery_test.go | 318 +++++++++++-- services/nvpair-ui-broker/manualnodes.go | 82 ++-- .../manualnodes_persistence_test.go | 10 +- services/nvpair-ui-broker/manualnodes_test.go | 225 +++++++++ services/nvpair-ui-broker/relay/relay.go | 32 +- services/nvpair-ui-broker/relay/relay_test.go | 42 +- services/nvpair-ui-broker/scheduler_test.go | 2 +- services/nvpair-ui-broker/telemetry.go | 21 +- services/nvpair-ui-broker/telemetry_test.go | 156 ++++++- .../nvpair-workload-manager/relaysource.go | 3 + .../relaysource_test.go | 90 +++- services/shared/noderec/noderec.go | 11 + services/tests/broker_management_test.go | 48 +- services/tests/cluster_data_plane_test.go | 68 ++- services/tests/remote_engine_test.go | 15 +- services/tests/secure_inference_test.go | 37 +- .../tests/workload_identity_interop_test.go | 4 +- services/tests/workload_interop_test.go | 96 ++-- 63 files changed, 3233 insertions(+), 1316 deletions(-) create mode 100644 desktop/tests/modular/empty-handlers.test.ts delete mode 100644 desktop/tests/modular/ghost-node-dedupe.test.ts create mode 100644 services/nvpair-cluster-manager/mdns_browser_test.go delete mode 100644 services/nvpair-manual-nodes/tls.go create mode 100644 services/nvpair-ui-broker/broker_test.go create mode 100644 services/nvpair-ui-broker/manualnodes_test.go diff --git a/desktop/docs/frontend-api.md b/desktop/docs/frontend-api.md index c33579d52..e409fd630 100644 --- a/desktop/docs/frontend-api.md +++ b/desktop/docs/frontend-api.md @@ -67,14 +67,17 @@ They do not imply a WebSocket connection. Browser clients are not supported. ### `pairApi.discovery` - `getNodes()` returns discoverable nodes. Each `AvailableNode` carries the - backend `id` (the stable per-host UUID that keys every domain), the display - `name` (hostname), the reachable `ipAddress`/`port`, and the `trusted` and - `clustered` flags. A `clustered` node already belongs to some cluster and is + backend `id` (the host UUID for verified rows, or the discovery record ID for + unverified rows), the display `name` (hostname), the reachable + `ipAddress`/`port`, and the `trusted` and `clustered` flags. A `clustered` node + already belongs to some cluster and is rendered non-invitable. Broker records also carry `discoveryRecordId` and `identityVerification` (`local`, `pinned`, or `unverified`). The bridge retains these provenance fields through proxy updates and clears them when the broker withdraws the record; public - hardware readings cannot change them. + hardware readings cannot change them. Verified rows require a matching + response `hostUuid` before accepting public hardware readings, including + local rows. Unverified previews may omit that field. - `onNodesChanged(callback)` reports discovery changes. ### `pairApi.engines` diff --git a/desktop/docs/services-parity.md b/desktop/docs/services-parity.md index 36bba35f6..72f3945b0 100644 --- a/desktop/docs/services-parity.md +++ b/desktop/docs/services-parity.md @@ -60,19 +60,20 @@ allows the user to restart the service. Personal AI Router consumes `discovery:get-nodes` and `discovery:nodes-changed` from the broker. The bridge: -- keys every node by the backend's stable per-host UUID (`AvailableNode.hostUuid`), +- keys verified rows by the backend's stable host UUID (`AvailableNode.id`), the same identity workloads (`originatedFrom`/`scheduledOn`), errors (`nodeId`), - proxy routing (`Node.ID`), `/v1/node-info` (`hostUuid`), and cluster membership - (`ClusterNode.nodeUuid`) all use; the hostname is display only; + proxy routing (`Node.ID`), and cluster membership (`ClusterNode.nodeUuid`) use; + unverified rows use their independent `discoveryRecordId`, so copied UUIDs, + hostnames, and addresses cannot merge them into verified rows; - preserves the backend's canonical reachable address; - reflects the discovery `trusted` (locally pinned peer) and `clustered` (belongs to some cluster) flags, and uses `clustered` to mark an already-clustered - discovered node non-invitable. `trusted` is a display annotation the scanner - re-derives whenever the cluster-manager reports a pin-set change, and again on - its periodic sweep of each peer's reported membership; it is not the routing - gate. Each proxy answers "do I hold a pin for this peer?" from its own - mesh when it resolves candidates, so PAIR must never treat `trusted` as - authoritative for whether work can reach a node; + discovered node non-invitable. Scanner derives `trusted` from current approved + pins after authenticating remote inventory. Discovery-fed proxy routing and + engine control require a local record or a trusted pinned peer; workload and + error synchronization require a trusted pinned peer. Protected network calls + also enforce the caller's current mesh trust. Explicitly configured engine + routes remain independent of discovery verification; - merges per-engine proxy presence; - carries flat and per-engine remote model inventories; - emits node and discovery pushes to the renderer. @@ -81,18 +82,17 @@ broker. The bridge: at startup and resolved as soon as the broker is ready, so self is identified by the same UUID key rather than a hostname guess. -The broker discovery surface does not include full dynamic telemetry, so -Electron polls `/v1/node-info` for CPU, memory, GPU, and VRAM values. +The broker's `discovery:node-info` stream supplies full hardware snapshots for +local and pinned records, matched to each row's current discovery record ID and +verification. Electron also polls public HTTP `/v1/node-info` for CPU, memory, +GPU, and VRAM displays, including unverified previews. Both paths update hardware +fields only. A public response for a verified row must include its matching +`hostUuid`; unverified previews may omit it. Public readings never change +identity, trust, service addresses, models, routing, or scheduling inputs. -Two limitations follow from the current discovery and node-info contract: - -- **Remote telemetry.** The poll is plain HTTP. A remote node that does not - answer `/v1/node-info` over plain HTTP contributes no live CPU/GPU/VRAM - telemetry; Personal AI Router still shows its discovery-level data. This resolves when the - backend exposes a plaintext metrics path for such nodes. -- **Remote OS.** The contract does not report a remote node's operating system. - The local node's OS is known from the running process; remote nodes fall back - to a placeholder until the backend reports OS on discovery or node-info. +The contract does not report a remote node's operating system. The local node's +OS is known from the running process; remote nodes fall back to a placeholder +until the backend reports OS on discovery or node-info. Manual nodes use the broker's `node/add`, `node/remove`, and `nodes/list` surface. Electron persists user entries and replays them after broker startup so @@ -492,7 +492,6 @@ provide an equivalent client-facing contract: | Relay the backend model catalogue to the renderer | `service-bridge/model-catalog.ts` | | Accumulate and reconcile receiver-side pending invites | `modular-state.ts`, `modular-supervisor.ts` | | Mirror backend-coupled runtime defaults not yet reported | `modular-runtime.ts` | -| Collapse a superseded node row before the scanner proves it | `modular-state.ts`, `modular-runtime.ts` | These are current integration boundaries. Do not add compatibility shims or duplicate backend services around them. diff --git a/desktop/src/electron/service-bridge/empty-handlers.ts b/desktop/src/electron/service-bridge/empty-handlers.ts index e00893226..655580e22 100644 --- a/desktop/src/electron/service-bridge/empty-handlers.ts +++ b/desktop/src/electron/service-bridge/empty-handlers.ts @@ -629,10 +629,10 @@ async function handleNodeRemoveMember( const selfId = state.getSelfId() const isSelfLeave = selfId !== null && payload.nodeId === selfId - // The broker owns manual-node persistence and resolves a stable node UUID - // back to the manual alias expected by nvpair-manual-nodes. For a purely - // discovered peer this remains a harmless no-op. - supervisor.callProcess('broker', 'node/remove', { id: payload.nodeId }).catch(() => {}) + // The broker resolves this node key to its manual entry, if any. Keeping it + // separate from the worker's entry ID prevents a matching user-supplied name + // from selecting a different configuration for removal. + supervisor.callProcess('broker', 'node/remove', { nodeId: payload.nodeId }).catch(() => {}) if (isSelfLeave) { // Self-departure is `cluster:leave`, not `nodes:remove` (the cluster-manager diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index f512de566..c8be77aa0 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -17,10 +17,7 @@ import type { ServiceError, ServiceErrorAction, ServiceErrorSeverity } from '@/s import type { Workload, WorkloadState } from '@/shared/types/workloads' import type { ClusterNode, Invite, NodeIdentityVerification } from '@/shared/types/cluster' import type { WsInvokeResponse } from '@/shared/types/ws-channels' -import { - MODULAR_NODE_INFO_SELF_HOST, - MODULAR_SUPERSEDE_MIN_AGE_MS -} from '@/shared/constants/modular-runtime' +import { MODULAR_NODE_INFO_SELF_HOST } from '@/shared/constants/modular-runtime' import { WorkloadStates } from '@/shared/constants/workloads' import { emptyEngineStatus, @@ -84,12 +81,11 @@ interface RemoteEngineFacts { interface ModularNode { discoveryRecordId?: string identityVerification?: NodeIdentityVerification - // Canonical cross-domain node key = the backend's stable per-host UUID - // (`shared/nodeid`). The broker carries it on `AvailableNode.hostUuid`, the - // proxies key `Node.ID` by it, and `/v1/node-info` reports it, so every - // domain now agrees on this id: workload `originatedFrom`/`scheduledOn`, - // error `nodeId`, and proxy routing all resolve against it. Never key, - // dedupe, or attribute by hostname — that is display only (`name`). + // Row key: host UUID for local/pinned hosts, discovery record ID for + // unverified discoveries. Proxy-only rows keep the proxy's operational key, + // which may identify a configured manual engine target. An unverified + // record cannot join a host's row just by claiming its UUID. The hostname + // is display only (`name`), never a correlation key. id: string sources: BrokerNodeSource[] // Display hostname (broker `AvailableNode.name` / proxy `Node.Host`). Shown @@ -701,8 +697,8 @@ function toEngineStatus( function parseProxyNode(params: JsonValue | undefined, engine: ProxyEngine): ModularNode | null { const obj = objectValue(params) if (!obj) return null - // The proxy keys `Node.ID` by the stable per-host UUID (and rejects empty), - // so this is already the canonical node key — no TXT parsing needed. + // The proxy supplies the operational node key: a host UUID or a separate + // configured manual target key. Use it directly, without parsing TXT claims. const id = stringValue(obj.id) if (!id) return null const engines = emptyEngines() @@ -763,9 +759,8 @@ function identityVerificationValue( function parseBrokerNode(params: JsonValue | undefined): ModularNode | null { const obj = objectValue(params) if (!obj) return null - // `AvailableNode.hostUuid` is the canonical node key. The broker's discovery - // store rejects an empty UUID, so every node it reports carries one; a - // payload without it is malformed and dropped rather than keyed by hostname. + // Verified hosts use their operational UUID. An unverified claim keeps its + // independent discovery record key, even when it claims that same UUID. const hostUuid = stringValue(obj.hostUuid) if (!hostUuid) return null // `AvailableNode.id`/`.name` are the mDNS instance hostname (display only). @@ -779,14 +774,14 @@ function parseBrokerNode(params: JsonValue | undefined): ModularNode | null { const discoveryRecordId = stringValue(obj.discoveryRecordId) const identityVerification = identityVerificationValue(obj.identityVerification) return { - id: hostUuid, + id: identityVerification === 'unverified' ? discoveryRecordId : hostUuid, ...(discoveryRecordId ? { discoveryRecordId } : {}), ...(identityVerification ? { identityVerification } : {}), sources: ['broker'], name: hostname, host: ipAddress, // Broker discovery is authoritative for trust/cluster membership state. - trusted: booleanValue(obj.trusted), + trusted: identityVerification !== 'unverified' && booleanValue(obj.trusted), clustered: booleanValue(obj.clustered), nodeInfoPort: port, nodeInfoUp: Boolean(ipAddress) && port > 0, @@ -1069,9 +1064,10 @@ class ModularBridgeState { /** * Fix the local node id to the cluster-manager's stable node UUID * (`cluster:get-node-id` → `nodeUuid`). This is the sole self-identity - * source: discovery and proxy ids are the same UUID key, so there is no - * hostname guessing to guard against. A no-op when unchanged; a change - * triggers a renderer refresh so per-node views re-resolve self. + * source: the local discovery and proxy rows use this UUID. Unverified + * discoveries keep separate record IDs and cannot establish self identity. + * A no-op when unchanged; a change triggers a renderer refresh so per-node + * views re-resolve self. */ setSelfId(nodeUuid: string): void { const trimmed = nodeUuid.trim() @@ -1166,6 +1162,10 @@ class ModularBridgeState { }) } + getNodeIdentityVerification(nodeId: string): NodeIdentityVerification | undefined { + return this.nodes.get(nodeId)?.identityVerification + } + getNodeInfoPollTargets(): { id: string; hosts: string[]; port: number }[] { const targets: { id: string; hosts: string[]; port: number }[] = [] for (const node of Array.from(this.nodes.values())) { @@ -1185,9 +1185,9 @@ class ModularBridgeState { continue } // Lead with the canonical display address, then retain the node's - // published alternatives. The poller walks them in order and verifies - // hostUuid, so a direct-connect address that resolves to another host - // cannot hide telemetry available at a later address. + // published alternatives. For verified rows, the poller requires a + // matching response host UUID and otherwise tries the next address. + // Unverified rows permit hardware previews without trusting UUID claims. const hosts = mergeAddresses([primaryNodeAddress(node)], nodeAddresses(node)) if (hosts.length === 0) continue targets.push({ id: node.id, hosts, port: node.nodeInfoPort }) @@ -1202,11 +1202,12 @@ class ModularBridgeState { const obj = objectValue(response) if (!obj) return - // `/v1/node-info` now reports the node's own hostUuid. If it disagrees - // with the node we polled (the address resolved to a different host), - // skip the merge rather than attribute another host's telemetry here. + // Outside unverified previews, require a matching response host UUID. + // An unverified row is keyed by its discovery record ID, so its public + // UUID claim need not match that key. Only hardware fields are merged; + // the response cannot change the row's broker-owned identity or trust. const reportedUuid = stringValue(obj.hostUuid) - if (reportedUuid && reportedUuid !== nodeId) return + if (node.identityVerification !== 'unverified' && reportedUuid !== nodeId) return const gpus = gpuArrayValue(obj.GPUs) const cpu = cpuValue(obj.cpu) @@ -1222,8 +1223,7 @@ class ModularBridgeState { gpus, cpu, memory, - inferenceHardwareIds, - nodeInfoUp: true + inferenceHardwareIds }) } @@ -2348,6 +2348,32 @@ class ModularBridgeState { } private handleBrokerNotification(notification: JsonRpcNotification): void { + if (notification.method === 'discovery:node-info') { + const snapshot = objectValue(notification.params) + if (!snapshot) return + const id = stringValue(snapshot.hostUuid) + const node = this.nodes.get(id) + const info = objectValue(snapshot.info) + if (!node || !info) { + // There must be an existing row and hardware payload to update. + return + } + if (node.discoveryRecordId !== stringValue(snapshot.discoveryRecordId)) { + // A shared host UUID cannot attach another record's snapshot. + return + } + if ( + node.identityVerification !== + identityVerificationValue(snapshot.identityVerification) || + node.identityVerification === 'unverified' + ) { + // Match the row's current verification state. Unverified rows + // cannot consume the broker's authenticated hardware stream. + return + } + this.mergeNodeInfoResponse(id, info) + return + } if (notification.method !== 'discovery:nodes-changed') return const nodes = brokerNodesValue(notification.params) @@ -2406,12 +2432,6 @@ class ModularBridgeState { // so there is no hostname guessing here — `setSelfId` is the sole source. const existing = this.nodes.get(node.id) const merged = this.mergeNode(existing, node, source) - // Collapse a wiped-and-rejoined host onto its new UUID before storing, - // so the pre-wipe record never reaches the map (see resolveAddressCollision). - if (this.resolveAddressCollision(merged)) { - this.removeNodeEntry(merged.id) - return - } if (existing && sameNode(existing, merged)) return this.nodes.set(merged.id, merged) this.emitNodeChanged(merged) @@ -2446,66 +2466,6 @@ class ModularBridgeState { } } - /** - * Collapse two UUIDs that are really one machine. - * - * A node whose appdata is wiped mints a fresh hostUuid, so peers advertise - * the same machine under a new key while the pre-wipe key survives — the - * fleet sees a duplicate "ghost" row. Nothing on the wire marks the two as - * the same host, so this infers it from the three signals the node-scanner - * uses to nominate a superseded record (`directory.supersedeCandidates`): - * same address, same hostname, and a `lastSeen` gap of at least - * MODULAR_SUPERSEDE_MIN_AGE_MS. - * - * It stops there, and the scanner does not. Those signals come from an - * unauthenticated mDNS advertisement, and `lastSeen` is not a liveness - * clock — the backend re-stamps a node only when its record CHANGES, so a - * healthy peer's `lastSeen` freezes exactly like a ghost's. The scanner - * therefore treats them as a suspicion and deletes only once node-info - * names a different host. This has no such proof available, so it must stay - * a display-side collapse: it hides a row until the next snapshot, and must - * never be read as the directory having evicted anything. - * - * Two caveats on how faithfully this can mirror the backend, both stemming - * from a proxy-sourced node carrying less than a discovery record does: - * `parseProxyNode` stamps `lastSeen` locally rather than echoing the - * backend's, and `primaryNodeAddress` falls back to `host` (a hostname for a - * proxy node), which can collapse the address and name conditions into one. - * Both only ever make this rule fire LESS often than the scanner's, so the - * node list stays a subset of the directory rather than contradicting it. - * - * Returns true when `node` is the STALE side and must not be stored — a - * backend that has not yet evicted the ghost keeps re-publishing it in every - * `discovery:nodes-changed` snapshot, so refusing it on arrival is what - * makes the eviction stick. Otherwise the losing entries are evicted here. - */ - private resolveAddressCollision(node: ModularNode): boolean { - const address = primaryNodeAddress(node) - if (!address || !node.name) return false - const rivals: ModularNode[] = [] - for (const other of this.nodes.values()) { - if (other.id === node.id || other.id === this.selfId) continue - if (primaryNodeAddress(other) !== address || other.name !== node.name) continue - rivals.push(other) - } - // Self is pinned by the cluster-manager's node UUID, so it is always the - // live identity for this machine's address: skipped above so it can - // never be evicted, and skipped as a claimant below so it never evicts. - // The backend applies the same asymmetry (directory.supersedeCandidates). - if (node.id === this.selfId) return false - // Decide before mutating. Returning mid-loop would strand entries - // already marked stale, leaving a partially applied collapse. - if (rivals.some(other => other.lastSeen - node.lastSeen >= MODULAR_SUPERSEDE_MIN_AGE_MS)) { - return true - } - for (const other of rivals) { - if (node.lastSeen - other.lastSeen >= MODULAR_SUPERSEDE_MIN_AGE_MS) { - this.removeNodeEntry(other.id) - } - } - return false - } - /** Delete a node and clean every index that referenced it. */ private removeNodeEntry(nodeId: string): void { if (!this.nodes.delete(nodeId)) return diff --git a/desktop/src/electron/service-bridge/node-info-poller.ts b/desktop/src/electron/service-bridge/node-info-poller.ts index ea83af152..5a19d8866 100644 --- a/desktop/src/electron/service-bridge/node-info-poller.ts +++ b/desktop/src/electron/service-bridge/node-info-poller.ts @@ -253,9 +253,10 @@ function withinWalkCooldown(choice: PollChoice | undefined, targetKey: string): } /** - * One attempt against one address, returning what it reported if it answered for - * this node. Probing is kept separate from {@link accept} so several addresses can - * be asked at once while only the best-ranked answer is the one used. + * Probe one address for display hardware. Rows keyed by a host UUID require a + * matching response UUID; unverified rows permit the preview under their own + * discovery record ID. Probing is separate from {@link accept} so several + * addresses can be asked at once while only the best-ranked answer is used. */ async function probeHost( nodeId: string, @@ -270,17 +271,22 @@ async function probeHost( if (run.aborted) return { parsed: null, reason: '' } if (!probe.parsed) return probe - // A reused/shared address can answer for another machine. That response is not - // success for this node, but neither should it stop failover to the remaining - // addresses the intended node published. + // A reused address may answer for another host. Outside unverified previews, + // skip that response and continue failover. An unverified row's key is a + // discovery record ID, so comparing its public UUID claim to that key would + // reject the intended preview. Neither path authenticates the HTTP response. const reportedUuid = nodeInfoHostUuid(probe.parsed) - if (reportedUuid && reportedUuid !== nodeId) { + const verification = getModularBridgeState().getNodeIdentityVerification(nodeId) + if (verification !== 'unverified' && reportedUuid !== nodeId) { log.verbose({ sublevel: 'node-info', - message: 'Node info candidate answered for a different host; trying next', + message: 'Node info candidate did not identify the expected host; trying next', data: { nodeId, reportedUuid, url: nodeInfoUrl(host, port) } }) - return { parsed: null, reason: `answered for ${reportedUuid}` } + return { + parsed: null, + reason: reportedUuid ? `answered for ${reportedUuid}` : 'missing hostUuid' + } } return probe } diff --git a/desktop/src/shared/constants/modular-runtime.ts b/desktop/src/shared/constants/modular-runtime.ts index 7486df965..f97968c70 100644 --- a/desktop/src/shared/constants/modular-runtime.ts +++ b/desktop/src/shared/constants/modular-runtime.ts @@ -92,19 +92,3 @@ export const MODULAR_CATALOG_CALL_TIMEOUT_MS = 30_000 // Also drives the pending-invite reconciliation sweep, the backstop for a missed // receiver-side `cluster:invite-canceled` / `cluster:invite-expired` push. export const MODULAR_INVITE_STATUS_POLL_INTERVAL_MS = 1_500 - -// Minimum `lastSeen` gap before one node record is treated as having superseded -// another that shares its address and hostname — a machine whose appdata was -// wiped rejoins under a fresh hostUuid and would otherwise show twice. -// -// Matches `supersedeMinAge` in `services/nvpair-node-scanner/directory.go` -// (300 seconds), but note the two are no longer the same rule. There, address, -// hostname and age only NOMINATE a record: they come from an unauthenticated -// mDNS advertisement, and a healthy peer's `lastSeen` freezes exactly as a -// ghost's does, so the scanner deletes nothing until node-info names a -// different host. The bridge collapses on the nomination alone, which is why -// this stays a display-side concern — it hides a row, it does not evict from -// the directory the TUI, the CLI, and every other consumer read. Hardcoded -// pending backend exposure — see -// docs/services-parity.md#current-client-side-responsibilities -export const MODULAR_SUPERSEDE_MIN_AGE_MS = 300_000 diff --git a/desktop/src/shared/types/cluster.ts b/desktop/src/shared/types/cluster.ts index 6d5f7d9d5..5b0d606a7 100644 --- a/desktop/src/shared/types/cluster.ts +++ b/desktop/src/shared/types/cluster.ts @@ -17,7 +17,11 @@ export interface AvailableNode { discoveryRecordId?: string /** How the service established this record's identity. */ identityVerification?: NodeIdentityVerification - /** Canonical node key = the backend's stable per-host UUID (matches `NodeItem.id`). */ + /** + * UI row key, matching `NodeItem.id`: host UUID for local/pinned records, + * discovery record ID for unverified records. Proxy-only rows retain the + * operational node key reported by the proxy. + */ id: string /** Display hostname. */ name: string @@ -58,9 +62,9 @@ export interface ClusterNode { id: string /** * Stable cryptographic identity; the trusted-node-store key and cert subject. - * This is the cross-domain correlation key — it equals a discovered node's - * hostUuid / `NodeItem.id` / `selfId`, so member↔node matching keys on this, - * never `id` (hostname). + * This matches a verified host's `NodeItem.id` and the local `selfId`. + * Unverified discoveries use separate record IDs even when they claim this + * UUID. Member-to-node matching uses this field, never `id` (hostname). */ nodeUuid: string name: string diff --git a/desktop/src/ui/hooks/useInvitablePeers.ts b/desktop/src/ui/hooks/useInvitablePeers.ts index 1e2fecda6..0f6ad3afa 100644 --- a/desktop/src/ui/hooks/useInvitablePeers.ts +++ b/desktop/src/ui/hooks/useInvitablePeers.ts @@ -23,9 +23,9 @@ export function useInvitablePeers(): AvailableNode[] { const selfId = useConnectionStore(state => state.selfId) return useMemo(() => { - // Correlate on the node UUID: `AvailableNode.id` = `ClusterNode.nodeUuid` - // = `selfId`. The hostname (`ClusterNode.id`) is display only and must - // never be used to match a discovered node against a member or self. + // Verified host rows match members and self by UUID. Unverified rows + // have separate discovery record IDs, so a copied UUID claim cannot + // make one disappear as a member or self. Hostnames are display only. const memberUuids = new Set(members.map(m => m.nodeUuid)) return discoveredNodes.filter(node => !memberUuids.has(node.id) && node.id !== selfId) }, [discoveredNodes, members, selfId]) diff --git a/desktop/tests/modular/empty-handlers.test.ts b/desktop/tests/modular/empty-handlers.test.ts new file mode 100644 index 000000000..bcc67fec6 --- /dev/null +++ b/desktop/tests/modular/empty-handlers.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from 'vitest' +import type { getEngineHubModels } from '@/electron/service-bridge/model-catalog' +import type { getModularSupervisor } from '@/electron/service-bridge/modular-supervisor' +import type { getModularBridgeState } from '@/electron/service-bridge/modular-state' + +type Supervisor = ReturnType +type BridgeState = ReturnType + +const mocks = vi.hoisted(() => ({ + state: { + getSelfId: vi.fn() + }, + supervisor: { + callProcess: vi.fn() + } +})) + +vi.mock('@/electron/service-bridge/modular-supervisor', () => ({ + getModularSupervisor: () => mocks.supervisor +})) +vi.mock('@/electron/service-bridge/modular-state', () => ({ + getModularBridgeState: () => mocks.state, + isUpstreamUnreachableError: () => false, + parseServiceErrors: () => [], + parseWorkloadsInitial: () => ({}) +})) +vi.mock('@/electron/service-bridge/model-catalog', () => ({ + getEngineHubModels: vi.fn() +})) + +import { handleServiceBridgeInvoke } from '@/electron/service-bridge/empty-handlers' + +describe('manual node removal', () => { + // The desktop knows a broker node key. Its request must select that key + // explicitly so an identical worker entry name cannot remove another entry. + it('identifies a broker node key separately from a manual entry ID', async () => { + const nodeId = 'manual:lab' + mocks.state.getSelfId.mockReturnValue('self') + mocks.supervisor.callProcess.mockResolvedValue({ nodeId, removed: true }) + + const result = await handleServiceBridgeInvoke('nodes:remove-member', { nodeId }) + + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith('broker', 'node/remove', { + nodeId + }) + expect(result).toEqual({ nodeId, removed: true }) + }) +}) diff --git a/desktop/tests/modular/ghost-node-dedupe.test.ts b/desktop/tests/modular/ghost-node-dedupe.test.ts deleted file mode 100644 index 29b8c6e08..000000000 --- a/desktop/tests/modular/ghost-node-dedupe.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it, vi } from 'vitest' - -vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) -vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) - -import { getModularBridgeState } from '@/electron/service-bridge/modular-state' -import { MODULAR_SUPERSEDE_MIN_AGE_MS } from '@/shared/constants/modular-runtime' - -const NOW = Date.now() -const STALE = NOW - 60 * 60 * 1000 -const RECENT = NOW - (MODULAR_SUPERSEDE_MIN_AGE_MS - 60_000) - -interface BrokerNodeFixture { - hostUuid: string - name: string - ipAddress: string - lastSeen: number -} - -function brokerSnapshot(nodes: BrokerNodeFixture[]): void { - getModularBridgeState().handleNotification({ - source: 'broker', - method: 'discovery:nodes-changed', - params: { - nodes: nodes.map(node => ({ - hostUuid: node.hostUuid, - name: node.name, - ipAddress: node.ipAddress, - port: 14318, - lastSeen: node.lastSeen, - trusted: false, - clustered: false - })) - } - }) -} - -function availableIds(): string[] { - return getModularBridgeState() - .getAvailableNodes() - .map(node => node.id) -} - -// A node whose appdata is wiped mints a fresh hostUuid, so the backend directory -// can carry BOTH the pre-wipe key and the new one for the same machine. The -// bridge collapses that pair on the three conditions the node-scanner uses to -// NOMINATE a superseded record (directory.supersedeCandidates) — same address, -// same hostname, and a lastSeen gap of at least MODULAR_SUPERSEDE_MIN_AGE_MS. -// -// The scanner goes on to confirm the nomination against node-info before it -// evicts anything; the bridge cannot, so what these cases pin down is a -// display-side collapse of a duplicate row, not an eviction from the directory. -describe('ghost node dedupe by address', () => { - it('drops the pre-wipe record when the same address and hostname reappear under a new UUID', () => { - brokerSnapshot([ - { - hostUuid: 'ghost-old-1', - name: 'wiped-host-1', - ipAddress: '198.51.100.10', - lastSeen: STALE - }, - { - hostUuid: 'ghost-new-1', - name: 'wiped-host-1', - ipAddress: '198.51.100.10', - lastSeen: NOW - } - ]) - - const ids = availableIds() - expect(ids).toContain('ghost-new-1') - expect(ids).not.toContain('ghost-old-1') - expect(getModularBridgeState().getNodesInitial().nodes['ghost-old-1']).toBeUndefined() - }) - - it('drops the pre-wipe record when it was already in the list first', () => { - brokerSnapshot([ - { - hostUuid: 'ghost-old-2', - name: 'wiped-host-2', - ipAddress: '198.51.100.11', - lastSeen: STALE - } - ]) - expect(availableIds()).toContain('ghost-old-2') - - brokerSnapshot([ - { - hostUuid: 'ghost-old-2', - name: 'wiped-host-2', - ipAddress: '198.51.100.11', - lastSeen: STALE - }, - { - hostUuid: 'ghost-new-2', - name: 'wiped-host-2', - ipAddress: '198.51.100.11', - lastSeen: NOW - } - ]) - - const ids = availableIds() - expect(ids).toContain('ghost-new-2') - expect(ids).not.toContain('ghost-old-2') - }) - - it('keeps the ghost suppressed across repeated snapshots that still carry it', () => { - const pair: BrokerNodeFixture[] = [ - { - hostUuid: 'ghost-old-3', - name: 'wiped-host-3', - ipAddress: '198.51.100.12', - lastSeen: STALE - }, - { - hostUuid: 'ghost-new-3', - name: 'wiped-host-3', - ipAddress: '198.51.100.12', - lastSeen: NOW - } - ] - // A backend that has not evicted the ghost yet keeps publishing it, so - // eviction only holds if the stale side is refused on every arrival, not - // just evicted once. - for (let i = 0; i < 3; i++) brokerSnapshot(pair) - - const ids = availableIds() - expect(ids.filter(id => id.startsWith('ghost-'))).toEqual(['ghost-new-3']) - }) - - it('collapses the pair regardless of order within a snapshot', () => { - brokerSnapshot([ - { - hostUuid: 'ghost-new-4', - name: 'wiped-host-4', - ipAddress: '198.51.100.13', - lastSeen: NOW - }, - { - hostUuid: 'ghost-old-4', - name: 'wiped-host-4', - ipAddress: '198.51.100.13', - lastSeen: STALE - } - ]) - - const ids = availableIds() - expect(ids).toContain('ghost-new-4') - expect(ids).not.toContain('ghost-old-4') - }) - - it('never collapses records whose hostnames differ, even on one address', () => { - // The hostname match is what makes this inference safe. A wipe clears - // identity but not the hostname; two records that disagree on the name - // are left for the scanner's identity probe to settle with proof. - brokerSnapshot([ - { - hostUuid: 'renamed-old-5', - name: 'old-name-5', - ipAddress: '198.51.100.14', - lastSeen: STALE - }, - { - hostUuid: 'renamed-new-5', - name: 'renamed-5', - ipAddress: '198.51.100.14', - lastSeen: NOW - } - ]) - - const ids = availableIds() - expect(ids).toContain('renamed-new-5') - expect(ids).toContain('renamed-old-5') - }) - - it('never collapses a pair whose lastSeen gap is under the minimum age', () => { - brokerSnapshot([ - { - hostUuid: 'recent-old-6', - name: 'shared-host-6', - ipAddress: '198.51.100.15', - lastSeen: RECENT - }, - { - hostUuid: 'recent-new-6', - name: 'shared-host-6', - ipAddress: '198.51.100.15', - lastSeen: NOW - } - ]) - - const ids = availableIds() - expect(ids).toContain('recent-new-6') - expect(ids).toContain('recent-old-6') - }) - - it('never collapses two distinct machines on different addresses', () => { - brokerSnapshot([ - { hostUuid: 'distinct-a', name: 'host-a', ipAddress: '198.51.100.20', lastSeen: STALE }, - { hostUuid: 'distinct-b', name: 'host-b', ipAddress: '198.51.100.21', lastSeen: NOW } - ]) - - const ids = availableIds() - expect(ids).toContain('distinct-a') - expect(ids).toContain('distinct-b') - }) - - it('never evicts the local node, and never hides a peer to protect it', () => { - const state = getModularBridgeState() - state.setSelfId('self-uuid-7') - brokerSnapshot([ - { - hostUuid: 'self-uuid-7', - name: 'self-host-7', - ipAddress: '198.51.100.30', - lastSeen: STALE - } - ]) - expect(availableIds()).toContain('self-uuid-7') - - // Self is pinned by the cluster-manager, so a peer claiming its address - // and hostname with a fresher timestamp must not displace it. The peer - // still shows: the backend keeps it in the directory too, and silently - // hiding it here is exactly the disagreement this rule exists to avoid. - brokerSnapshot([ - { - hostUuid: 'self-uuid-7', - name: 'self-host-7', - ipAddress: '198.51.100.30', - lastSeen: STALE - }, - { - hostUuid: 'peer-7', - name: 'self-host-7', - ipAddress: '198.51.100.30', - lastSeen: NOW - } - ]) - - const ids = availableIds() - expect(ids).toContain('self-uuid-7') - expect(ids).toContain('peer-7') - }) - - it('applies a multi-way collapse completely or not at all', () => { - // Three records on one address and hostname: the stale side must not - // strand entries it had already marked for eviction. - brokerSnapshot([ - { - hostUuid: 'multi-old-a', - name: 'shared-host-8', - ipAddress: '198.51.100.40', - lastSeen: STALE - }, - { - hostUuid: 'multi-old-b', - name: 'shared-host-8', - ipAddress: '198.51.100.40', - lastSeen: STALE - }, - { - hostUuid: 'multi-new', - name: 'shared-host-8', - ipAddress: '198.51.100.40', - lastSeen: NOW - } - ]) - - const ids = availableIds().filter(id => id.startsWith('multi-')) - expect(ids).toEqual(['multi-new']) - }) -}) diff --git a/desktop/tests/modular/node-identity-contract.test.ts b/desktop/tests/modular/node-identity-contract.test.ts index 5d1fd7560..4ff3c983e 100644 --- a/desktop/tests/modular/node-identity-contract.test.ts +++ b/desktop/tests/modular/node-identity-contract.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from 'vitest' +import type { NodeIdentityVerification } from '@/shared/types/cluster' vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) @@ -9,16 +10,212 @@ vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) import { getModularBridgeState } from '@/electron/service-bridge/modular-state' describe('node identity contract', () => { + // Shared names, addresses, and different observation ages cannot merge + // distinct verified hosts, even when the same snapshot is received again. + it('preserves distinct records with the same name and address across repeated snapshots', () => { + const oldHostUuid = 'old-peer' + const newHostUuid = 'new-peer' + const sharedName = 'shared-name' + const sharedAddress = '192.0.2.20' + const state = getModularBridgeState() + const params = [ + { + hostUuid: oldHostUuid, + discoveryRecordId: 'peer:old-peer', + identityVerification: 'pinned', + name: sharedName, + ipAddress: sharedAddress, + lastSeen: 1 + }, + { + hostUuid: newHostUuid, + discoveryRecordId: 'peer:new-peer', + identityVerification: 'pinned', + name: sharedName, + ipAddress: sharedAddress, + lastSeen: 1790000000 + } + ] + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params + }) + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params + }) + + expect(state.getAvailableNodes().map(node => node.id)).toEqual([oldHostUuid, newHostUuid]) + }) + + // Seed accepted hardware, then vary one identity field at a time. Each + // rejection must protect the accepted data without relying on another guard. + describe('broker hardware snapshots', () => { + const hostUuid = 'snapshot-peer' + const discoveryRecordId = 'peer:snapshot-peer' + const acceptedGpuName = 'Accepted GPU' + const updatedGpuName = 'Updated GPU' + const test = ( + name: string, + snapshotRecordId: string, + snapshotVerification: NodeIdentityVerification, + expectedGpuName: string + ): void => { + it(name, () => { + const state = getModularBridgeState() + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [ + { + hostUuid, + discoveryRecordId, + identityVerification: 'pinned', + name: 'verified', + ipAddress: '192.0.2.20', + port: 14318, + trusted: true + } + ] + }) + state.handleNotification({ + source: 'broker', + method: 'discovery:node-info', + params: { + hostUuid, + discoveryRecordId, + identityVerification: 'pinned', + info: { + hostUuid, + GPUs: [{ name: acceptedGpuName }] + } + } + }) + expect( + state.getNodesInitial().nodes[hostUuid].topology.gpus.map(gpu => gpu.name) + ).toEqual([acceptedGpuName]) + + state.handleNotification({ + source: 'broker', + method: 'discovery:node-info', + params: { + hostUuid, + discoveryRecordId: snapshotRecordId, + identityVerification: snapshotVerification, + info: { + hostUuid, + GPUs: [{ name: updatedGpuName }] + } + } + }) + + expect( + state.getNodesInitial().nodes[hostUuid].topology.gpus.map(gpu => gpu.name) + ).toEqual([expectedGpuName]) + }) + } + + test( + 'updates hardware for the matching verified record', + discoveryRecordId, + 'pinned', + updatedGpuName + ) + test( + 'rejects hardware from a different discovery record', + 'peer:another-record', + 'pinned', + acceptedGpuName + ) + test( + 'rejects hardware with different identity verification', + discoveryRecordId, + 'unverified', + acceptedGpuName + ) + }) + + // Render a verified peer and a candidate copying its identity. Public + // hardware may populate the candidate's preview without merging the rows + // or accepting the response's forged name and verification claims. + it('keeps an unverified UUID claim separate from a verified peer and permits its hardware preview', () => { + const hostUuid = 'peer-a' + const candidateRecordId = 'candidate:rogue' + const peerName = 'peer' + const peerAddress = '192.0.2.20' + const nodeInfoPort = 14318 + const state = getModularBridgeState() + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [ + { + hostUuid, + discoveryRecordId: 'peer:peer-a', + identityVerification: 'pinned', + name: peerName, + ipAddress: peerAddress, + port: nodeInfoPort, + trusted: true + }, + { + hostUuid, + discoveryRecordId: candidateRecordId, + identityVerification: 'unverified', + name: peerName, + ipAddress: '192.0.2.30', + port: nodeInfoPort + } + ] + }) + + state.mergeNodeInfoResponse(candidateRecordId, { + hostUuid: 'different-public-claim', + GPUs: [{ name: 'Display GPU' }], + peer: { name: 'forged-name' }, + identityVerification: 'pinned' + }) + + expect(state.getAvailableNodes().map(node => node.id)).toEqual( + expect.arrayContaining([hostUuid, candidateRecordId]) + ) + expect(state.getAvailableNodes().find(node => node.id === hostUuid)).toMatchObject({ + ipAddress: peerAddress, + trusted: true + }) + expect(state.getAvailableNodes().find(node => node.id === candidateRecordId)).toMatchObject( + { identityVerification: 'unverified', name: peerName } + ) + expect(state.getNodesInitial().nodes[candidateRecordId].topology.gpus).toHaveLength(1) + + state.mergeNodeInfoResponse(candidateRecordId, { + GPUs: [{ name: 'UUID-less preview GPU' }] + }) + + expect( + state.getNodesInitial().nodes[candidateRecordId].topology.gpus.map(gpu => gpu.name) + ).toEqual(['UUID-less preview GPU']) + expect(state.getAvailableNodes().find(node => node.id === candidateRecordId)).toMatchObject( + { identityVerification: 'unverified', name: peerName } + ) + }) + // Let discovery and a proxy both report one host, then withdraw discovery. + // The proxy may keep the row visible, but cannot retain broker verification. it('clears withdrawn broker provenance while proxy presence survives', () => { + const hostUuid = 'identity-withdrawn-host' + const hostName = 'workstation' + const address = '192.0.2.21' const state = getModularBridgeState() state.handleNotification({ source: 'broker', method: 'discovery:nodes-changed', params: [ { - hostUuid: 'identity-withdrawn-host', - name: 'workstation', - ipAddress: '192.0.2.21', + hostUuid, + name: hostName, + ipAddress: address, port: 14318, discoveryRecordId: 'peer:identity-withdrawn-host', identityVerification: 'pinned' @@ -29,10 +226,10 @@ describe('node identity contract', () => { source: 'ollama-proxy', method: 'node/discovered', params: { - id: 'identity-withdrawn-host', - host: 'workstation', + id: hostUuid, + host: hostName, port: 11434, - addresses: ['192.0.2.21'] + addresses: [address] } }) state.handleNotification({ @@ -41,24 +238,30 @@ describe('node identity contract', () => { params: [] }) - const node = state.getAvailableNodes().find(node => node.id === 'identity-withdrawn-host') + const node = state.getAvailableNodes().find(node => node.id === hostUuid) expect(node).toBeDefined() expect(node).not.toHaveProperty('discoveryRecordId') expect(node).not.toHaveProperty('identityVerification') }) - it('preserves broker provenance through proxy and public hardware updates', () => { + // A proxy's host UUID and an unverified discovery record identify separate + // rows. Public hardware may fill the candidate without changing its identity. + it('keeps a candidate separate from proxy presence and ignores public identity claims', () => { + const hostUuid = 'identity-contract-host' + const discoveryRecordId = 'candidate:identity-contract' + const hostName = 'workstation' + const address = '192.0.2.20' const state = getModularBridgeState() state.handleNotification({ source: 'broker', method: 'discovery:nodes-changed', params: [ { - hostUuid: 'identity-contract-host', - name: 'workstation', - ipAddress: '192.0.2.20', + hostUuid, + name: hostName, + ipAddress: address, port: 14318, - discoveryRecordId: 'observation:candidate-a', + discoveryRecordId, identityVerification: 'unverified' } ] @@ -67,27 +270,28 @@ describe('node identity contract', () => { source: 'ollama-proxy', method: 'node/discovered', params: { - id: 'identity-contract-host', - host: 'workstation', + id: hostUuid, + host: hostName, port: 11434, - addresses: ['192.0.2.20'] + addresses: [address] } }) - state.mergeNodeInfoResponse('identity-contract-host', { - hostUuid: 'identity-contract-host', + expect(state.getAvailableNodes().map(node => node.id)).toEqual( + expect.arrayContaining([hostUuid, discoveryRecordId]) + ) + state.mergeNodeInfoResponse(discoveryRecordId, { + hostUuid, discoveryRecordId: 'peer:identity-contract-host', identityVerification: 'pinned', GPUs: [{ name: 'Display GPU' }] }) - expect( - state.getAvailableNodes().find(node => node.id === 'identity-contract-host') - ).toMatchObject({ - discoveryRecordId: 'observation:candidate-a', - identityVerification: 'unverified' - }) - expect(state.getNodesInitial().nodes['identity-contract-host'].topology.gpus).toHaveLength( - 1 + expect(state.getAvailableNodes().find(node => node.id === discoveryRecordId)).toMatchObject( + { + discoveryRecordId, + identityVerification: 'unverified' + } ) + expect(state.getNodesInitial().nodes[discoveryRecordId].topology.gpus).toHaveLength(1) }) }) diff --git a/desktop/tests/modular/node-info-poll-cycle.test.ts b/desktop/tests/modular/node-info-poll-cycle.test.ts index 7cc5dca3a..773bab783 100644 --- a/desktop/tests/modular/node-info-poll-cycle.test.ts +++ b/desktop/tests/modular/node-info-poll-cycle.test.ts @@ -9,12 +9,14 @@ import { MODULAR_NODE_INFO_WALK_COOLDOWN_MS } from '@/shared/constants/modular-runtime' import type { getModularBridgeState } from '@/electron/service-bridge/modular-state' +import type { JsonValue } from '@/electron/service-bridge/json-rpc-subprocess' type BridgeState = ReturnType const mocks = vi.hoisted(() => ({ state: { getNodeInfoPollTargets: vi.fn(), + getNodeIdentityVerification: vi.fn(), mergeNodeInfoResponse: vi.fn() }, logged: [] as { @@ -121,6 +123,7 @@ describe('node info polling', () => { vi.stubGlobal('fetch', fetchMock) fetchMock.mockReset() mocks.state.getNodeInfoPollTargets.mockReturnValue([]) + mocks.state.getNodeIdentityVerification.mockReturnValue(undefined) mocks.logged.length = 0 }) @@ -130,6 +133,89 @@ describe('node info polling', () => { vi.useRealTimers() }) + // A candidate row is keyed by its discovery record ID, so its public UUID + // need not match that key. Deliver the response to the requested preview + // row without treating the claimed UUID as authenticated identity. + const preview = (name: string, response: JsonValue): void => { + it(name, async () => { + const discoveryRecordId = 'candidate:display' + const address = '127.0.0.1' + const nodeInfoPort = 14318 + mocks.state.getNodeInfoPollTargets.mockReturnValue([ + { + id: discoveryRecordId, + hosts: [address], + port: nodeInfoPort + } + ]) + mocks.state.getNodeIdentityVerification.mockReturnValue('unverified') + fetchMock.mockResolvedValue(new Response(JSON.stringify(response), { status: 200 })) + + startNodeInfoPoller() + await vi.advanceTimersByTimeAsync(0) + + expect(mocks.state.mergeNodeInfoResponse).toHaveBeenCalledWith( + discoveryRecordId, + response + ) + }) + } + preview('merges public hardware into the unverified record despite its claimed UUID', { + hostUuid: 'a-public-claim' + }) + preview('merges public hardware into an unverified preview without hostUuid', { + GPUs: [{ name: 'Preview GPU' }] + }) + + // Recheck the current record after HTTP completes so a removed discovery + // cannot retain the preview exception from when its request started. + it('does not accept an unverified preview after its record disappears during the request', async () => { + const discoveryRecordId = 'candidate:removed-during-poll' + mocks.state.getNodeInfoPollTargets.mockReturnValue([ + { id: discoveryRecordId, hosts: ['127.0.0.1'], port: 14318 } + ]) + mocks.state.getNodeIdentityVerification.mockReturnValue('unverified') + const response = Promise.withResolvers() + fetchMock.mockReturnValue(response.promise) + + startNodeInfoPoller() + await vi.advanceTimersByTimeAsync(0) + mocks.state.getNodeIdentityVerification.mockReturnValue(undefined) + mocks.state.getNodeInfoPollTargets.mockReturnValue([]) + response.resolve(new Response(JSON.stringify({ GPUs: [{ name: 'Old preview GPU' }] }))) + await vi.advanceTimersByTimeAsync(0) + + expect(mocks.state.mergeNodeInfoResponse).not.toHaveBeenCalled() + }) + + it('tries the next address when a verified response omits hostUuid', async () => { + const hostUuid = 'verified-missing-identity' + const firstAddress = '192.0.2.20' + const nextAddress = '192.0.2.21' + const nodeInfoPort = 14318 + mocks.state.getNodeInfoPollTargets.mockReturnValue([ + { id: hostUuid, hosts: [firstAddress, nextAddress], port: nodeInfoPort } + ]) + mocks.state.getNodeIdentityVerification.mockReturnValue('pinned') + fetchMock.mockImplementation(input => + Promise.resolve( + String(input).includes(firstAddress) + ? new Response(JSON.stringify({ GPUs: [{ name: 'Unidentified GPU' }] }), { + status: 200 + }) + : answersFor(hostUuid) + ) + ) + + startNodeInfoPoller() + await vi.advanceTimersByTimeAsync(0) + + expect(mocks.state.mergeNodeInfoResponse).toHaveBeenCalledExactlyOnceWith(hostUuid, { + hostUuid + }) + expect(outageWarnings()).toHaveLength(0) + }) + it('reaches a working address past ones that drop connections', async () => { mocks.state.getNodeInfoPollTargets.mockReturnValue([ { diff --git a/desktop/tests/modular/uuid-node-keying.test.ts b/desktop/tests/modular/uuid-node-keying.test.ts index 700929712..1784d658e 100644 --- a/desktop/tests/modular/uuid-node-keying.test.ts +++ b/desktop/tests/modular/uuid-node-keying.test.ts @@ -11,11 +11,10 @@ import { parseWorkloadsInitial } from '@/electron/service-bridge/modular-state' -// The bridge keys every node by the backend's stable per-host UUID; the hostname -// is display only. These tests lock in the two behaviours that migration exists -// for: a broker view and a proxy view of the same machine collapse to one -// UUID-keyed entry (no hostname ghost), and cross-domain attribution refuses a -// telemetry response whose reported UUID does not match the polled node. +// These cases cover rows keyed by the backend's stable host UUID, with hostnames +// used only for display. Broker and proxy views of one host share a row, and a +// response naming another UUID is rejected. Unverified discovery rows instead +// use independent record IDs, covered in node-identity-contract.test.ts. describe('UUID node keying', () => { it('merges a broker node and a same-hostUuid proxy node into one UUID-keyed entry', () => { const state = getModularBridgeState() @@ -26,6 +25,8 @@ describe('UUID node keying', () => { nodes: [ { hostUuid: 'uuid-merge-1', + discoveryRecordId: 'peer:uuid-merge-1', + identityVerification: 'pinned', name: 'merge-host-1', ipAddress: '192.0.2.11', port: 14318, @@ -52,10 +53,17 @@ describe('UUID node keying', () => { // No hostname-keyed ghost — that double-entry is exactly what UUID keying fixes. expect(nodes['merge-host-1']).toBeUndefined() expect(nodes['uuid-merge-1'].name).toBe('merge-host-1') + // The proxy port proves that engine presence reached the existing row. + expect(nodes['uuid-merge-1'].port).toBe(11434) - // Broker trust/cluster flags survive the proxy merge (the proxy feed has none). + // Broker identity and trust survive the proxy merge (the proxy feed has neither). const available = state.getAvailableNodes().find(node => node.id === 'uuid-merge-1') - expect(available).toMatchObject({ trusted: true, clustered: true }) + expect(available).toMatchObject({ + discoveryRecordId: 'peer:uuid-merge-1', + identityVerification: 'pinned', + trusted: true, + clustered: true + }) // Clustered peers need authoritative facts before run state is inferred, even // after the proxy presence is merged onto the UUID-keyed entry. expect(state.isRemoteEngineRunning('uuid-merge-1', 'ollama')).toBe(false) @@ -141,6 +149,39 @@ describe('UUID node keying', () => { expect(state.getNodesInitial().nodes['uuid-ni-1'].topology.cpu.model).toBe('CPU-A') }) + const rejectMissingIdentity = (name: string, verification: 'local' | 'pinned'): void => { + it(name, () => { + const hostUuid = `uuid-missing-identity-${verification}` + const state = getModularBridgeState() + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [ + { + hostUuid, + discoveryRecordId: `${verification}:${hostUuid}`, + identityVerification: verification, + name: 'verified host', + ipAddress: '192.0.2.51', + port: 14318 + } + ] + }) + state.mergeNodeInfoResponse(hostUuid, { + hostUuid, + GPUs: [{ name: 'Verified host GPU' }] + }) + + state.mergeNodeInfoResponse(hostUuid, { GPUs: [{ name: 'Unidentified GPU' }] }) + + expect( + state.getNodesInitial().nodes[hostUuid].topology.gpus.map(gpu => gpu.name) + ).toEqual(['Verified host GPU']) + }) + } + rejectMissingIdentity('preserves local hardware when a response omits hostUuid', 'local') + rejectMissingIdentity('preserves pinned hardware when a response omits hostUuid', 'pinned') + it('gives the node-info poller every published address in ranked order', () => { const state = getModularBridgeState() state.handleNotification({ diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 3981ef5d0..73df3975f 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -646,8 +646,8 @@ enriches node records with details from two services: Authenticated node-info supplies the canonical peer's name, addresses, and registered ports as well as hardware. Scanner replaces only the successful discovery with this verified record. Other observations remain independently -unverified. Consumer merge and replay enforcement remain subsequent work in the -[scanner specification](../services/nvpair-node-scanner/spec.md). +unverified. The broker retains record provenance through caches and relays; +unverified claims cannot replace service endpoints or supply scheduling data. Each enrichment keeps a last-good value, so one failed fetch dims nothing. A node's card does not blank out because a single poll timed out. @@ -681,8 +681,9 @@ identity reports and open TCP ports cannot remove or retain verified peers. Some networks block or filter multicast, so discovery is not the only path in. `nvpair-manual-nodes` takes an address you enter directly and probes it on a fixed interval, and a manual node that answers is folded into the same directory -as a discovered one. It is initially keyed by the address you typed, and re-keyed -to the peer's real UUID as soon as that node reports it. +as a discovered one. A manual configuration has its own key until a pinned mTLS +read proves the peer's UUID. Public hardware responses remain display-only; +explicit manual engine configuration does not authorize their use for scheduling. ## Trust Boundaries diff --git a/services/nvpair-cluster-manager/mdns_browser.go b/services/nvpair-cluster-manager/mdns_browser.go index 2950d7195..1d2d1c32a 100644 --- a/services/nvpair-cluster-manager/mdns_browser.go +++ b/services/nvpair-cluster-manager/mdns_browser.go @@ -33,17 +33,16 @@ func newBrowser() *Browser { return &Browser{relay: make(map[string]discovery.Node)} } -// setRelay replaces the resolver map from a discovery:nodes snapshot: every node -// advertising cl with an IP becomes a resolvable entry keyed by hostUuid (the -// rest are dropped). The broker sends the full filtered set on every change, so -// this is a wholesale replace, not a per-node apply — a departed node is simply -// absent from the next snapshot and stops resolving. The synthesized TXT carries -// uuid= so Resolve's UUID match works (hostUuid is the same principal -// the node advertises as uuid=), plus the node's own address ranking so callers -// can fail over past an address they cannot reach. +// setRelay replaces the resolver map from a discovery:nodes snapshot. Records +// with a cluster service and dialable addresses remain selectable by record ID +// for pairing. Verified records take precedence when resolving a host UUID. +// Departed records disappear with the next snapshot. Synthesized TXT preserves +// address ranking for callers that need address failover. func (b *Browser) setRelay(nodes []noderec.DirectoryNode) { m := make(map[string]discovery.Node, len(nodes)) + verified := make(map[string]bool, len(nodes)) for _, n := range nodes { + proved := n.IdentityVerification == noderec.IdentityLocal || n.IdentityVerification == noderec.IdentityPinned svc, ok := n.Services[noderec.ServiceCluster] if !ok { continue @@ -55,12 +54,22 @@ func (b *Browser) setRelay(nodes []noderec.DirectoryNode) { if len(hosts) == 0 { continue } - m[n.HostUUID] = discovery.Node{ + node := discovery.Node{ ID: n.Name, Addresses: hosts, Port: svc.Port, - TXT: append([]string{"uuid=" + n.HostUUID}, n.AddressTXT()...), + TXT: n.AddressTXT(), } + // A selected discovery remains addressable for pairing, independently + // of the preferred endpoint for an already verified host. + if n.DiscoveryRecordID != "" { + m[n.DiscoveryRecordID] = node + } + if verified[n.HostUUID] && !proved { + continue + } + m[n.HostUUID] = node + verified[n.HostUUID] = proved } b.mu.Lock() b.relay = m @@ -73,30 +82,20 @@ func (b *Browser) setRelay(nodes []noderec.DirectoryNode) { // ranking is evidence from its vantage point, not a promise about this host's. func (b *Browser) Resolve(idOrUUID string) (hosts []string, port int, ok bool) { b.mu.RLock() - nodes := make([]discovery.Node, 0, len(b.relay)) - for _, n := range b.relay { - nodes = append(nodes, n) + defer b.mu.RUnlock() + // setRelay and seed index every UUID directly. Only hostname lookups need + // a scan, so adding another writer must preserve that indexing invariant. + if node, present := b.relay[idOrUUID]; present && node.Port > 0 { + if addresses := pickHosts(node); len(addresses) > 0 { + return addresses, node.Port, true + } } - b.mu.RUnlock() - - var byNode discovery.Node - haveNode := false - for _, n := range nodes { - if n.Port == 0 { + for _, n := range b.relay { + if n.ID != idOrUUID || n.Port == 0 { continue } - if discovery.UUIDFromTXT(n.TXT) == idOrUUID { - if h := pickHosts(n); len(h) > 0 { - return h, n.Port, true - } - } - if n.ID == idOrUUID && !haveNode { - byNode, haveNode = n, true - } - } - if haveNode { - if h := pickHosts(byNode); len(h) > 0 { - return h, byNode.Port, true + if addresses := pickHosts(n); len(addresses) > 0 { + return addresses, n.Port, true } } return nil, 0, false @@ -119,6 +118,5 @@ func (b *Browser) seed(uuid, host string, port int) { ID: uuid, Addresses: []string{host}, Port: port, - TXT: []string{"uuid=" + uuid}, } } diff --git a/services/nvpair-cluster-manager/mdns_browser_test.go b/services/nvpair-cluster-manager/mdns_browser_test.go new file mode 100644 index 000000000..ab0f693ad --- /dev/null +++ b/services/nvpair-cluster-manager/mdns_browser_test.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "nvpair-shared/noderec" +) + +// A candidate copying a verified UUID must still be selectable for pairing by +// its discovery record ID. UUID lookup must keep returning the verified peer's +// endpoint, regardless of which record arrives first. +func TestBrowser_ResolvesDiscoveryRecordsWithoutReplacingVerifiedHost(t *testing.T) { + const peerUUID = "peer" + const peerAddress = "192.0.2.20" + const candidateRecordID = "candidate:rogue" + const candidateAddress = "192.0.2.30" + const clusterPort = 14321 + peer := noderec.DirectoryNode{ + HostUUID: peerUUID, + DiscoveryRecordID: "peer:peer", + IdentityVerification: noderec.IdentityPinned, + IP: peerAddress, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceCluster: {Port: clusterPort}, + }, + } + candidate := peer + candidate.DiscoveryRecordID = candidateRecordID + candidate.IdentityVerification = noderec.IdentityUnverified + candidate.IP = candidateAddress + test := func(name string, records []noderec.DirectoryNode) { + t.Run(name, func(t *testing.T) { + browser := newBrowser() + browser.setRelay(records) + + addresses, port, ok := browser.Resolve(candidateRecordID) + require.True(t, ok) + assert.Equal(t, []string{candidateAddress}, addresses) + assert.Equal(t, clusterPort, port) + addresses, _, ok = browser.Resolve(peerUUID) + require.True(t, ok) + assert.Equal(t, []string{peerAddress}, addresses) + }) + } + test("peer arrives first", []noderec.DirectoryNode{peer, candidate}) + test("candidate arrives first", []noderec.DirectoryNode{candidate, peer}) +} + +func TestBrowser_ResolvesRecordUUIDAndHostnameWithRankedAddresses(t *testing.T) { + const peerUUID = "peer-uuid" + const recordID = "peer:peer-uuid" + const hostName = "peer-workstation" + const primaryAddress = "192.168.240.1" + const secondaryAddress = "10.172.55.129" + const clusterPort = 14321 + browser := newBrowser() + browser.setRelay([]noderec.DirectoryNode{{ + HostUUID: peerUUID, DiscoveryRecordID: recordID, Name: hostName, + IdentityVerification: noderec.IdentityPinned, + IP: primaryAddress, + IPs: []string{primaryAddress, secondaryAddress}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceCluster: {Port: clusterPort}}, + }}) + test := func(name, key string) { + t.Run(name, func(t *testing.T) { + addresses, port, found := browser.Resolve(key) + + require.True(t, found) + assert.Equal(t, []string{primaryAddress, secondaryAddress}, addresses) + assert.Equal(t, clusterPort, port) + }) + } + test("discovery record ID", recordID) + test("host UUID", peerUUID) + test("hostname", hostName) +} diff --git a/services/nvpair-engine-manager/remote_test.go b/services/nvpair-engine-manager/remote_test.go index 490f48dca..acb37f5f2 100644 --- a/services/nvpair-engine-manager/remote_test.go +++ b/services/nvpair-engine-manager/remote_test.go @@ -36,7 +36,12 @@ func TestRunRemoteNotClustered(t *testing.T) { // pinned dial can't be built. m := NewManager(NewCodec(&out), &Executor{progress: newProgressHub()}, nil) m.peers.set([]noderec.DirectoryNode{{ - HostUUID: "uuid-b", Name: "nodeB", IP: "192.168.1.42", ClusterUUID: "cuuid-b", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: "uuid-b", + Name: "nodeB", + IP: "192.168.1.42", + ClusterUUID: "cuuid-b", Services: map[noderec.ServiceKey]noderec.ServiceStatus{ noderec.ServiceEngineControl: {Port: 14323}, }, diff --git a/services/nvpair-engine-manager/remotepeers.go b/services/nvpair-engine-manager/remotepeers.go index a75bf6e69..f3688e822 100644 --- a/services/nvpair-engine-manager/remotepeers.go +++ b/services/nvpair-engine-manager/remotepeers.go @@ -39,14 +39,31 @@ func newPeerDirectory() *peerDirectory { return &peerDirectory{peers: make(map[string]ecPeer)} } -// set replaces the directory from a discovery:nodes snapshot, keeping only nodes -// that advertise the ec service and a dialable address. +// set replaces the directory from a discovery:nodes snapshot, accepting local +// records and trusted pinned peers with a usable engine-control endpoint. func (d *peerDirectory) set(nodes []noderec.DirectoryNode) { next := make(map[string]ecPeer, len(nodes)) for _, n := range nodes { + verification := n.IdentityVerification + if verification != noderec.IdentityLocal && verification != noderec.IdentityPinned { + // Discovery claims alone cannot select a destination for engine + // commands, even when they copy another peer's UUID and trusted flag. + continue + } + if verification == noderec.IdentityPinned && !n.Trusted { + // Remote control requires current trust in addition to proved peer + // identity. The owned local record does not require a peer pin. + continue + } svc, ok := n.Services[noderec.ServiceEngineControl] addresses := n.CandidateIPs() - if !ok || len(addresses) == 0 || svc.Port == 0 { + if !ok || svc.Port == 0 { + // Without an advertised control service and port there is no + // engine-control listener to contact on this node. + continue + } + if len(addresses) == 0 { + // An eligible identity still needs an address for the connection. continue } // Key by the stable hostUuid so a target survives a rename and same-named diff --git a/services/nvpair-engine-manager/remotepeers_test.go b/services/nvpair-engine-manager/remotepeers_test.go index 949322e67..f8bc01cf4 100644 --- a/services/nvpair-engine-manager/remotepeers_test.go +++ b/services/nvpair-engine-manager/remotepeers_test.go @@ -12,43 +12,141 @@ import ( "nvpair-shared/noderec" ) +// Present a verified engine-control peer followed by an unverified copy with +// another address. Building the peer directory must retain the verified endpoint +// so a later discovery claim cannot redirect engine commands. +func TestPeerDirectory_UnverifiedClaimsCannotReplaceVerifiedEndpoints(t *testing.T) { + const peerUUID = "peer" + const peerAddress = "192.0.2.20" + directory := newPeerDirectory() + peer := noderec.DirectoryNode{ + HostUUID: peerUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + IP: peerAddress, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceEngineControl: {Port: 14323}, + }, + } + candidate := peer + candidate.IdentityVerification = noderec.IdentityUnverified + candidate.IP = "192.0.2.30" + + directory.set([]noderec.DirectoryNode{peer, candidate}) + + resolved, ok := directory.lookup(peerUUID) + require.True(t, ok) + assert.Equal(t, []string{peerAddress}, resolved.addresses) +} + +// A record with a complete engine-control endpoint must still be absent from +// the directory unless it has both proved peer identity and current trust. +func TestPeerDirectory_RejectsUnverifiedOrUntrustedPeers(t *testing.T) { + const hostUUID = "peer" + test := func(name string, verification noderec.IdentityVerification, trusted bool) { + t.Run(name, func(t *testing.T) { + directory := newPeerDirectory() + node := noderec.DirectoryNode{ + HostUUID: hostUUID, + IP: "192.0.2.20", + IdentityVerification: verification, + Trusted: trusted, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceEngineControl: {Port: 14323}, + }, + } + + directory.set([]noderec.DirectoryNode{node}) + + _, present := directory.lookup(hostUUID) + assert.False(t, present) + }) + } + test("unverified identity with claimed trust", noderec.IdentityUnverified, true) + test("missing verification with claimed trust", "", true) + test("pinned identity without current trust", noderec.IdentityPinned, false) +} + +// Locally owned engine-control records remain available without cluster +// membership. Remote peer trust is not required for the local worker's record. +func TestPeerDirectory_AcceptsLocalRecordsWithoutPeerTrust(t *testing.T) { + const hostUUID = "self" + const controlPort = 14323 + directory := newPeerDirectory() + node := noderec.DirectoryNode{ + HostUUID: hostUUID, + IP: "127.0.0.1", + IdentityVerification: noderec.IdentityLocal, + Trusted: false, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceEngineControl: {Port: controlPort}, + }, + } + + directory.set([]noderec.DirectoryNode{node}) + + peer, present := directory.lookup(hostUUID) + require.True(t, present) + assert.Equal(t, hostUUID, peer.nodeID) + assert.Equal(t, controlPort, peer.port) +} + func TestPeerDirectory_ReplacesEntriesAndResolvesPeers(t *testing.T) { + const peerUUID = "uuid-b" + const noControlUUID = "uuid-c" + const noAddressUUID = "uuid-d" + const primaryAddress = "192.168.1.42" + const secondaryAddress = "10.0.0.42" + const clusterUUID = "cuuid-b" + const controlPort = 14323 d := newPeerDirectory() d.set([]noderec.DirectoryNode{ { - HostUUID: "uuid-b", Name: "nodeB", IP: "192.168.1.42", - IPs: []string{"192.168.1.42", "10.0.0.42"}, ClusterUUID: "cuuid-b", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: peerUUID, + Name: "nodeB", + IP: primaryAddress, + IPs: []string{primaryAddress, secondaryAddress}, + ClusterUUID: clusterUUID, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceEngineControl: {Port: 14323}, + noderec.ServiceEngineControl: {Port: controlPort}, }, }, { // advertises em but not ec -> excluded - HostUUID: "uuid-c", Name: "nodeC", IP: "192.168.1.43", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: noControlUUID, + Name: "nodeC", + IP: "192.168.1.43", Services: map[noderec.ServiceKey]noderec.ServiceStatus{ noderec.ServiceEngineManager: {Port: 14322}, }, }, { // ec but no dialable IP -> excluded - HostUUID: "uuid-d", Name: "nodeD", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: noAddressUUID, + Name: "nodeD", Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceEngineControl: {Port: 14323}, + noderec.ServiceEngineControl: {Port: controlPort}, }, }, }) - p, ok := d.lookup("uuid-b") + p, ok := d.lookup(peerUUID) require.True(t, ok, "unexpected nodeB entry (%v, %v)", p, ok) - assert.Equal(t, []string{"192.168.1.42", "10.0.0.42"}, p.addresses, "unexpected nodeB addresses") - require.Equal(t, 14323, p.port, "unexpected nodeB entry (%v, %v)", p, ok) - require.Equal(t, "cuuid-b", p.clusterUUID, "unexpected nodeB entry (%v, %v)", p, ok) - _, ok = d.lookup("uuid-c") + assert.Equal(t, []string{primaryAddress, secondaryAddress}, p.addresses, "unexpected nodeB addresses") + require.Equal(t, controlPort, p.port, "unexpected nodeB entry (%v, %v)", p, ok) + require.Equal(t, clusterUUID, p.clusterUUID, "unexpected nodeB entry (%v, %v)", p, ok) + _, ok = d.lookup(noControlUUID) require.False(t, ok, "nodeC advertises no ec; should not be in directory") - _, ok = d.lookup("uuid-d") + _, ok = d.lookup(noAddressUUID) require.False(t, ok, "nodeD has no dialable IP; should not be in directory") // A later snapshot replaces the set wholesale. d.set(nil) - _, ok = d.lookup("uuid-b") + _, ok = d.lookup(peerUUID) require.False(t, ok, "empty snapshot should clear the directory") } @@ -57,17 +155,25 @@ func TestPeerDirectory_ReplacesEntriesAndResolvesPeers(t *testing.T) { // a rename and same-named peers stay distinct. A record without a // hostUuid falls back to the instance name. func TestPeerDirectory_KeysByHostUUID(t *testing.T) { + const hostUUID = "uuid-b" + const hostName = "nodeB" + const address = "192.168.1.42" d := newPeerDirectory() d.set([]noderec.DirectoryNode{{ - HostUUID: "uuid-b", Name: "nodeB", IP: "192.168.1.42", ClusterUUID: "cuuid-b", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: hostUUID, + Name: hostName, + IP: address, + ClusterUUID: "cuuid-b", Services: map[noderec.ServiceKey]noderec.ServiceStatus{ noderec.ServiceEngineControl: {Port: 14323}, }, }}) - p, ok := d.lookup("uuid-b") + p, ok := d.lookup(hostUUID) require.True(t, ok, "expected lookup by hostUuid (%v, %v)", p, ok) - require.Equal(t, "uuid-b", p.nodeID, "expected lookup by hostUuid (%v, %v)", p, ok) - assert.Equal(t, []string{"192.168.1.42"}, p.addresses, "expected lookup by hostUuid") - _, ok = d.lookup("nodeB") + require.Equal(t, hostUUID, p.nodeID, "expected lookup by hostUuid (%v, %v)", p, ok) + assert.Equal(t, []string{address}, p.addresses, "expected lookup by hostUuid") + _, ok = d.lookup(hostName) require.False(t, ok, "must not be addressable by hostname when a hostUuid is present") } diff --git a/services/nvpair-errors/manager.go b/services/nvpair-errors/manager.go index 8d4f37e06..9f339bfc1 100644 --- a/services/nvpair-errors/manager.go +++ b/services/nvpair-errors/manager.go @@ -508,9 +508,9 @@ func (m *Manager) emitRelayEvent(ev DiscoveryEvent) { } } -// directoryToPeer projects a relay DirectoryNode onto the errors service's er -// peer, returning false when it doesn't advertise er or has no address. The nodeId is -// the peer's stable hostUuid, matching the UUID each peer stamps as the origin +// directoryToPeer projects a trusted pinned record onto an error-sync peer, +// rejecting records without the errors service or a dialable address. The nodeId +// is the peer's stable hostUuid, matching the UUID each peer stamps as the origin // of its errors and uses as its own localNodeID — so the self-check and // EvictNode key on an identity that survives a peer's PC rename. A // relay DirectoryNode always carries a hostUuid (the scanner guarantees it), so @@ -518,6 +518,9 @@ func (m *Manager) emitRelayEvent(ev DiscoveryEvent) { // cluster-uuid= is reconstructed into TXT so PeerSync's mTLS pin lookup is // unchanged. func directoryToPeer(n noderec.DirectoryNode) (RawNode, bool) { + if n.IdentityVerification != noderec.IdentityPinned || !n.Trusted { + return RawNode{}, false + } svc, ok := n.Services[noderec.ServiceErrors] addresses := n.CandidateIPs() if !ok || len(addresses) == 0 { diff --git a/services/nvpair-errors/manager_test.go b/services/nvpair-errors/manager_test.go index 27ffc80fa..52a2c54e1 100644 --- a/services/nvpair-errors/manager_test.go +++ b/services/nvpair-errors/manager_test.go @@ -4,6 +4,7 @@ package main import ( + "context" "encoding/json" "fmt" "io" @@ -12,8 +13,64 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "nvpair-shared/discovery" + "nvpair-shared/noderec" ) +// A claimed trusted flag and an error-service port are insufficient without +// verified identity. Such a record must not become an error-sync peer. +func TestDirectoryToPeer_RejectsUnverifiedServiceClaims(t *testing.T) { + node := noderec.DirectoryNode{ + HostUUID: "peer", + IdentityVerification: noderec.IdentityUnverified, + Trusted: true, + IP: "192.0.2.30", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceErrors: {Port: 14319}, + }, + } + _, accepted := directoryToPeer(node) + assert.False(t, accepted) +} + +func TestDiscovery_RemovesPeersWhenTrustIsWithdrawn(t *testing.T) { + const peerUUID = "peer" + const peerAddress = "192.0.2.20" + m, _ := newTestManager(t) + m.ctx = context.Background() + events := make(chan DiscoveryEvent, 2) + m.SetRelayEvents(events) + peer := noderec.DirectoryNode{ + HostUUID: peerUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + IP: peerAddress, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceErrors: {Port: 14319}, + }, + } + applySnapshot := func(node noderec.DirectoryNode) { + t.Helper() + params, err := json.Marshal(noderec.GetNodesResult{Nodes: []noderec.DirectoryNode{node}}) + require.NoError(t, err) + m.handleMessage(&Message{JSONRPC: "2.0", Method: noderec.NotifyNodes, Params: params}) + } + applySnapshot(peer) + require.Len(t, events, 1) + discovered := <-events + require.Equal(t, discovery.Discovered, discovered.Type) + assert.Equal(t, peerUUID, discovered.Node.ID) + assert.Equal(t, []string{peerAddress}, discovered.Node.Addresses) + + peer.Trusted = false + applySnapshot(peer) + + require.Len(t, events, 1) + removed := <-events + assert.Equal(t, discovery.Removed, removed.Type) + assert.Equal(t, peerUUID, removed.Node.ID) +} + // captureRW splits the manager's outbound stream into two channels at // write time: frames that carry a JSON-RPC `id` (responses, including // errors) land in `responses`, frames without an `id` (notifications, diff --git a/services/nvpair-errors/rename_test.go b/services/nvpair-errors/rename_test.go index 1999fe592..1f75a8347 100644 --- a/services/nvpair-errors/rename_test.go +++ b/services/nvpair-errors/rename_test.go @@ -27,18 +27,26 @@ func (nopReadWriter) Write(p []byte) (int, error) { return len(p), nil } // never conflate two same-named peers. Host stays the hostname for dialing / // display. func TestDirectoryToPeer_KeysByHostUUID(t *testing.T) { + const hostUUID = "uuid-x" + const hostName = "host-x" + const primaryAddress = "10.0.0.4" + const secondaryAddress = "192.168.1.4" n := noderec.DirectoryNode{ - HostUUID: "uuid-x", - Name: "host-x", - IP: "10.0.0.4", - IPs: []string{"10.0.0.4", "192.168.1.4"}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceErrors: {Port: 14319}}, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: hostUUID, + Name: hostName, + IP: primaryAddress, + IPs: []string{primaryAddress, secondaryAddress}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceErrors: {Port: 14319}, + }, } p, ok := directoryToPeer(n) require.True(t, ok, "node advertising er with an IP should project") - assert.Equal(t, "uuid-x", p.ID, "peer ID") - assert.Equal(t, "host-x", p.Host, "peer Host") - assert.Equal(t, []string{"10.0.0.4", "192.168.1.4"}, p.Addresses, "peer addresses") + assert.Equal(t, hostUUID, p.ID, "peer ID") + assert.Equal(t, hostName, p.Host, "peer Host") + assert.Equal(t, []string{primaryAddress, secondaryAddress}, p.Addresses, "peer addresses") } // TestSetLocalNodeID: the broker's --node-id override replaces the hostname diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index 52c5de7b1..e7f5fdc57 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -16,10 +16,7 @@ Uses bidirectional newline-delimited JSON-RPC 2.0. By default, communication is | Flag | Default | Description | |---|---|---| | `--ipc ` | _(stdio)_ | IPC endpoint: Unix socket or Windows named pipe | -| `--client-cert ` | _(none)_ | PEM client certificate to present when probing TLS-enabled manual nodes (requires `--client-key`) | -| `--client-key ` | _(none)_ | PEM client private key matching `--client-cert` | -| `--ca-bundle ` | _(none)_ | PEM bundle of CAs to trust when verifying server certificates (additive to the system trust store) | -| `--cluster-dir ` | _(none)_ | Cluster config dir; when set, a TLS manual node's node-info is probed over cluster mTLS, presenting this node's leaf and accepting any currently-pinned server cert | +| `--cluster-dir ` | _(none)_ | Current cluster identity and exact certificate pins, required for all service node-info reads | | `--log-level ` | `info` | `error` / `warn` / `info` / `debug`; falls back to `$NVPAIR_LOG_LEVEL`. Also changeable at runtime via the `log/set-level` method | | `--version` | | Print version and exit | @@ -55,7 +52,8 @@ Emitted when a manually added node has been probed and its initial status determ "gpus":[{"name":"NVIDIA GeForce RTX 3080","utilization_percent":37}], "telemetryValid":true, "msSince":120, - "hostUuid":"stable-node-uuid" + "hostUuid":"stable-node-uuid", + "identityVerification":"pinned" } } ``` @@ -64,8 +62,8 @@ Each node is probed for both inference engines: Ollama on its default `:11434` ( ### `node/updated` -Emitted when a periodic probe detects a change (service going up/down, models, -GPU values, telemetry validity, or sample age changed). `telemetryValid` and +Emitted when a periodic probe detects a change, and for every authenticated +hardware sample even when values are unchanged. `telemetryValid` and `msSince` preserve node-info's distinction between an idle 0% sample and missing telemetry so the broker can feed manual nodes into the same scheduler cache. @@ -93,7 +91,7 @@ A hostname is preferred over an IP literal: probe clients disable keep-alives sp |---|---|---| | `address` | Yes | IP address or hostname of the node, with no port | | `name` | No | Friendly name (used as node ID; defaults to `manual:
`) | -| `tls_port` | No | Probe node-info over HTTPS on this port instead of plain HTTP on `14318`. Echoed back as `tls_enabled` | +| `tls_port` | No | Override the node-info port (default `14318`). Hardware reads always require pinned mTLS. Echoed back as `tls_enabled` | | `mtls` | No | Stored and echoed back as `mtls_required`. The probe transport itself is chosen by `tls_port` and live cluster membership, so this field records intent rather than driving it | Response: the initial node status object. @@ -138,7 +136,13 @@ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, fo - **LM Studio** on port 1234: `GET /v1/models`, which doubles as the liveness check and the model list - **Node Info** on port 14318, or `tls_port` over HTTPS: hardware inventory and identity (`GET /v1/node-info`) -A node can have any combination of these, or none if the target is unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. +A node can have any combination of these, or none if the target is unreachable. +Hardware reads require exact current cluster pins and matching `hostUuid` and +`clusterUuid` in the response. They use pooled mTLS clients after identifying +the peer, refuse redirects, and never fall back to public HTTP or operator CAs. +Unverified configurations report `identityVerification: "unverified"` and supply +no service measurements. Explicitly configured engine routes remain available. +Desktop and TUI may independently read public HTTP for hardware display. The three engine ports are compiled in: only the node-info leg's port can be moved, via `tls_port`. A remote engine on a non-default port is not discovered. diff --git a/services/nvpair-manual-nodes/main.go b/services/nvpair-manual-nodes/main.go index 3334b5abb..83cd5deb4 100644 --- a/services/nvpair-manual-nodes/main.go +++ b/services/nvpair-manual-nodes/main.go @@ -20,10 +20,7 @@ import ( func main() { ipcPath := flag.String("ipc", "", "IPC endpoint: Unix domain socket path or Windows named pipe (default: stdin/stdout)") - clientCert := flag.String("client-cert", "", "path to PEM client certificate to present when probing TLS-enabled manual nodes") - clientKey := flag.String("client-key", "", "path to PEM client private key matching --client-cert") - caBundle := flag.String("ca-bundle", "", "path to PEM bundle of CAs to trust for verifying server certificates (additive to system trust store)") - clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); when set, a TLS manual node is probed over cluster mTLS with our pinned leaf") + clusterDir := flag.String("cluster-dir", "", "cluster identity and current certificate pins for authenticated node-info reads") showVersion := flag.Bool("version", false, "print version and exit") resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo) flag.Parse() @@ -35,22 +32,9 @@ func main() { applog.Init("nvpair-manual-nodes", resolveLevel()) - // While this node is a cluster member, a TLS manual node is probed over - // cluster mTLS (the only way to reach a clustered peer's node-info); - // otherwise the BYO --client-cert / plaintext paths are unchanged. The mesh is - // a live view of the cluster dir, so probes start using the cluster leaf as - // soon as this node joins, without a restart. + // The live mesh follows cluster changes without restarting the worker. mesh := clustertrust.Open(*clusterDir) - tlsOpts := tlsClientOptions{ - CertPath: *clientCert, - KeyPath: *clientKey, - CABundlePath: *caBundle, - } - if err := tlsOpts.validate(); err != nil { - log.Fatalf("invalid TLS configuration: %v", err) - } - var transport io.ReadWriteCloser if *ipcPath != "" { conn, err := dialIPC(*ipcPath) @@ -80,10 +64,7 @@ func main() { }() codec := NewCodec(transport) - mgr, err := NewManager(codec, tlsOpts, mesh) - if err != nil { - log.Fatalf("failed to construct manager: %v", err) - } + mgr := NewManager(codec, mesh) if err := mgr.Run(ctx); err != nil && ctx.Err() == nil { log.Fatalf("manager error: %v", err) diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 55a4040aa..9b326885a 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -19,6 +19,7 @@ import ( "nvpair-shared/applog" "nvpair-shared/clustertrust" "nvpair-shared/errors" + "nvpair-shared/noderec" ) // Version is stamped at build time via -ldflags "-X main.Version=...". @@ -77,17 +78,16 @@ type NodeInfoResponse struct { // identity as the rest of the fleet — including deduping with the same // machine if it's also discovered over mDNS. Empty when the remote // predates this field or isn't a NVPAIR node-info server. - HostUUID string `json:"hostUuid,omitempty"` + HostUUID string `json:"hostUuid,omitempty"` + ClusterUUID *string `json:"clusterUuid,omitempty"` } -// ManualEntry is the user-supplied identity of a manually added -// node. Address + Name are the historical fields. TLSPort is an -// optional hint that node-info on this host listens for HTTPS on -// the given port (typically 14319 with our default node-info -// flags); a zero TLSPort means "probe plain HTTP on 14318" — the -// historical behavior. MTLS is informational only on the entry -// (we don't probe differently), but it's persisted so the UI can -// show the operator the configuration they set. +// ManualEntry identifies a user-configured endpoint by its bare hostname or IP +// address and optional name. TLSPort overrides the node-info port, defaulting +// to 14318 when zero. Hardware probes always use HTTPS with current cluster +// pins and our client certificate, regardless of TLSPort or MTLS. MTLS is an +// informational configuration flag echoed in status, not a transport switch. +// The separate Ollama and LM Studio engine probes use plain HTTP. type ManualEntry struct { Address string `json:"address"` Name string `json:"name"` @@ -95,20 +95,18 @@ type ManualEntry struct { MTLS bool `json:"mtls,omitempty"` } -// ManualNodeStatus mirrors a manual entry plus the latest probe -// outcome. TLSEnabled is the entry-level configuration echoed to -// the UI so the node card can render a lock icon without having to -// cross-reference the entries list. NodeInfoPort reflects whichever -// port we actually reached (TLS port if TLS is configured, plain -// 14318 otherwise) so existing UI that displays the port stays -// correct. +// ManualNodeStatus combines a manual entry with its latest probe results. +// TLSEnabled and MTLSRequired echo the entry's configuration hints for the UI; +// they do not describe whether the hardware probe uses TLS (it always does). +// NodeInfoPort is the attempted HTTPS port, even when that probe fails. type ManualNodeStatus struct { - ID string `json:"id"` - Name string `json:"name,omitempty"` - Address string `json:"address"` - OllamaUp bool `json:"ollama_up"` - OllamaPort int `json:"ollama_port"` - OllamaModels []string `json:"ollama_models,omitempty"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + ID string `json:"id"` + Name string `json:"name,omitempty"` + Address string `json:"address"` + OllamaUp bool `json:"ollama_up"` + OllamaPort int `json:"ollama_port"` + OllamaModels []string `json:"ollama_models,omitempty"` // LM Studio is probed on its default OpenAI-API port the same way Ollama // is on 11434, so a manually-added node running LM Studio can be bridged // into lmstudio-proxy by a supervising broker. @@ -130,6 +128,26 @@ type ManualNodeStatus struct { HostUUID string `json:"hostUuid,omitempty"` } +// sameAvailability compares identity, configuration, engine models, and service +// availability. Every authenticated hardware sample is published independently. +func (s ManualNodeStatus) sameAvailability(other ManualNodeStatus) bool { + return s.IdentityVerification == other.IdentityVerification && + s.ID == other.ID && + s.Name == other.Name && + s.Address == other.Address && + s.OllamaUp == other.OllamaUp && + s.OllamaPort == other.OllamaPort && + sliceEqual(s.OllamaModels, other.OllamaModels) && + s.LMStudioUp == other.LMStudioUp && + s.LMStudioPort == other.LMStudioPort && + sliceEqual(s.LMStudioModels, other.LMStudioModels) && + s.NodeInfoUp == other.NodeInfoUp && + s.NodeInfoPort == other.NodeInfoPort && + s.TLSEnabled == other.TLSEnabled && + s.MTLSRequired == other.MTLSRequired && + s.HostUUID == other.HostUUID +} + type ReadyParams struct { Version string `json:"version"` } @@ -147,50 +165,40 @@ type trackedNode struct { consecutiveFails int } +// Manager keeps manual entries in memory and publishes their probe results. +// The supervising broker owns durable configuration and downstream routing. type Manager struct { codec *Codec cancel context.CancelFunc - // client is the plain-HTTP probe client (Ollama on :11434 and - // nvpair-node-info's HTTP listener). tlsClient is used only when - // the entry sets TLSPort > 0; if the operator hasn't given us - // any TLS material via flags, tlsClient still works because - // the default *http.Client respects the system trust store - // (which covers public CAs and any roots an enterprise has - // installed). client_test.go reaches in and replaces the - // transports on both for unit tests, so they're plain - // *http.Client values rather than something more elaborate. - client *http.Client - tlsClient *http.Client - - // mesh is non-nil when this node is clustered (--cluster-dir): a TLS manual - // node is probed over cluster mTLS (presenting our leaf, accepting any - // currently-pinned server cert) since a clustered peer's node-info is - // pin-gated and serves no plaintext listener. + // client probes Ollama and LM Studio over plain HTTP. Fresh connections + // let hostname entries recover after their remote address changes. + client *http.Client + + // mesh supplies current membership, our client certificate, and exact + // peer pins. Hardware probes require these even without a TLS port hint. mesh *clustertrust.Mesh + // peerClients reuses clients bound to a previously verified host UUID. + // Unknown endpoints first use mesh.ClientTLSConfigAny to learn which + // pinned peer answered, and probeNodeInfo checks that binding afterward. + peerClients *clustertrust.PeerClientPool + mu sync.RWMutex nodes map[string]*trackedNode } -func NewManager(codec *Codec, tlsOpts tlsClientOptions, mesh *clustertrust.Mesh) (*Manager, error) { - tlsClient, err := buildTLSClient(tlsOpts, probeTimeout) - if err != nil { - return nil, fmt.Errorf("build TLS client: %w", err) - } - if tlsClient == nil { - tlsClient = &http.Client{Timeout: probeTimeout, Transport: noKeepAliveTransport()} - } +func NewManager(codec *Codec, mesh *clustertrust.Mesh) *Manager { return &Manager{ - codec: codec, - mesh: mesh, + codec: codec, + mesh: mesh, + peerClients: clustertrust.NewPeerClientPool(mesh, probeTimeout), client: &http.Client{ Timeout: probeTimeout, Transport: noKeepAliveTransport(), }, - tlsClient: tlsClient, - nodes: make(map[string]*trackedNode), - }, nil + nodes: make(map[string]*trackedNode), + } } // noKeepAliveTransport returns a probe transport with idle-connection reuse @@ -207,6 +215,7 @@ func (m *Manager) Run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) m.cancel = cancel defer cancel() + defer m.peerClients.CloseIdle() if err := m.codec.Notify("ready", ReadyParams{Version: Version}); err != nil { return fmt.Errorf("failed to send ready notification: %w", err) @@ -247,6 +256,11 @@ func (m *Manager) probeAll(ctx context.Context) { } } +// probeNode refreshes Ollama and LM Studio reachability and attempts a pinned +// mTLS node-info read for one manual entry. It saves the combined status if the +// entry is still tracked, publishes changes and every accepted hardware sample, +// and reports sustained outages. An answering engine alone does not verify the +// host's identity or authorize its hardware data for scheduling. func (m *Manager) probeNode(entry ManualEntry) { addr := entry.Address id := nodeID(entry) @@ -254,52 +268,64 @@ func (m *Manager) probeNode(entry ManualEntry) { ollamaUp, ollamaModels := m.probeOllama(addr, 11434) lmStudioUp, lmStudioModels := m.probeLMStudio(addr, lmStudioPort) - // Pick scheme + port + client based on the entry's TLS hint. - // The operator decides which scheme this manual node uses; we - // don't probe both. TLSPort > 0 means HTTPS on that port via - // the TLS client (which carries the operator's client cert, - // if configured). Otherwise it's plain HTTP on the historical - // 14318. - scheme := "http" + // The entry selects a port, not a protocol: even the default 14318 is + // probed with pinned HTTPS, never through the public HTTP hardware path. nodeInfoPort := 14318 - probeClient := m.client if entry.TLSPort > 0 { - scheme = "https" nodeInfoPort = entry.TLSPort - probeClient = m.tlsClient - // Clustered: a TLS manual node is a cluster peer whose node-info is - // pin-gated mTLS with no plaintext listener. Dial it with our cluster - // leaf, accepting any currently-pinned server cert (a manual node has no - // cluster-uuid= TXT to key a specific pin on). Refresh first so a cluster - // joined, or a peer paired, after startup is seen; falls back to the BYO - // tlsClient while unclustered. - m.mesh.Refresh() - if cfg, ok := m.mesh.ClientTLSConfigAny(); ok { - probeClient = &http.Client{Timeout: probeTimeout, Transport: &http.Transport{TLSClientConfig: cfg, DisableKeepAlives: true}} + } + // Refresh membership before selecting a client. A previously verified entry + // uses its host's exact pin; an unknown endpoint may present any current + // pin, then must bind its response identities to that certificate. + m.mesh.Refresh() + nodeInfoUp, info := false, NodeInfoResponse{} + m.mu.RLock() + var knownUUID string + if current := m.nodes[id]; current != nil && current.status.IdentityVerification == noderec.IdentityPinned { + knownUUID = current.status.HostUUID + } + m.mu.RUnlock() + if client, ok := m.peerClients.Client(knownUUID); knownUUID != "" && ok { + nodeInfoUp, info = m.probeNodeInfo(client, addr, nodeInfoPort) + } else if cfg, ok := m.mesh.ClientTLSConfigAny(); ok { + client := &http.Client{ + Timeout: probeTimeout, + Transport: &http.Transport{ + TLSClientConfig: cfg, + DisableKeepAlives: true, + }, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, } + nodeInfoUp, info = m.probeNodeInfo(client, addr, nodeInfoPort) } - nodeInfoUp, info := m.probeNodeInfo(probeClient, scheme, addr, nodeInfoPort) + m.peerClients.DropUnpinned() newStatus := ManualNodeStatus{ - ID: id, - Name: entry.Name, - Address: addr, - OllamaUp: ollamaUp, - OllamaPort: 11434, - OllamaModels: ollamaModels, - LMStudioUp: lmStudioUp, - LMStudioPort: lmStudioPort, - LMStudioModels: lmStudioModels, - NodeInfoUp: nodeInfoUp, - NodeInfoPort: nodeInfoPort, - TLSEnabled: entry.TLSPort > 0, - MTLSRequired: entry.TLSPort > 0 && entry.MTLS, - GPUs: info.GPUs, - CPU: info.CPU, - Memory: info.Memory, - TelemetryValid: info.TelemetryValid, - MSSince: info.MSSince, - HostUUID: info.HostUUID, + IdentityVerification: noderec.IdentityUnverified, + ID: id, + Name: entry.Name, + Address: addr, + OllamaUp: ollamaUp, + OllamaPort: 11434, + OllamaModels: ollamaModels, + LMStudioUp: lmStudioUp, + LMStudioPort: lmStudioPort, + LMStudioModels: lmStudioModels, + NodeInfoUp: nodeInfoUp, + NodeInfoPort: nodeInfoPort, + TLSEnabled: entry.TLSPort > 0, + MTLSRequired: entry.TLSPort > 0 && entry.MTLS, + GPUs: info.GPUs, + CPU: info.CPU, + Memory: info.Memory, + TelemetryValid: info.TelemetryValid, + MSSince: info.MSSince, + HostUUID: info.HostUUID, + } + if nodeInfoUp { + newStatus.IdentityVerification = noderec.IdentityPinned } reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.NodeInfoUp @@ -313,13 +339,8 @@ func (m *Manager) probeNode(entry ManualEntry) { prev := tn.status prevFails := tn.consecutiveFails - // A failed node-info probe returns no HostUUID; preserve the last-learned one - // so a transient node-info blip (while Ollama stays reachable) doesn't rekey - // a live node UUID -> manual-id -> UUID — the broker keys the discovery store - // by HostUUID. - if !nodeInfoUp && newStatus.HostUUID == "" { - newStatus.HostUUID = prev.HostUUID - } + // A failed authenticated read clears service measurements and proof. The + // configured engine target remains independently available when it answers. tn.status = newStatus if reachable { tn.consecutiveFails = 0 @@ -329,29 +350,23 @@ func (m *Manager) probeNode(entry ManualEntry) { curFails := tn.consecutiveFails m.mu.Unlock() - changed := prev.OllamaUp != newStatus.OllamaUp || - prev.LMStudioUp != newStatus.LMStudioUp || - prev.NodeInfoUp != newStatus.NodeInfoUp || - prev.HostUUID != newStatus.HostUUID || - !sliceEqual(prev.OllamaModels, newStatus.OllamaModels) || - !sliceEqual(prev.LMStudioModels, newStatus.LMStudioModels) || - !gpusEqual(prev.GPUs, newStatus.GPUs) || - !cpuEqual(prev.CPU, newStatus.CPU) || - !memoryEqual(prev.Memory, newStatus.Memory) || - prev.TelemetryValid != newStatus.TelemetryValid || - prev.MSSince != newStatus.MSSince + changed := !prev.sameAvailability(newStatus) if changed { slog.Info("manual node state changed", "node_id", id, "addr", addr, "ollama_up", newStatus.OllamaUp, "node_info_up", newStatus.NodeInfoUp, "models", len(newStatus.OllamaModels), "gpus", len(newStatus.GPUs)) - m.codec.Notify("node/updated", newStatus) } else { slog.Debug("manual node probe stable", "node_id", id, "addr", addr, "ollama_up", newStatus.OllamaUp, "node_info_up", newStatus.NodeInfoUp) } + if changed || nodeInfoUp { + if err := m.codec.Notify("node/updated", newStatus); err != nil { + slog.Warn("failed to publish manual node status", "node_id", id, "err", err) + } + } // Surface the consecutive-failure transition through the errors // pipeline. The supervising broker (dispatchSubprocessErrorsNotif) @@ -496,33 +511,76 @@ func (m *Manager) fetchOllamaModels(addr string, port int) []string { return names } -func (m *Manager) probeNodeInfo(client *http.Client, scheme, addr string, port int) (bool, NodeInfoResponse) { - url := scheme + "://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/node-info" +// probeNodeInfo reads a bounded HTTPS hardware snapshot and binds its reported +// host and principal to the response certificate and current cluster pins. +// Success means the remote identity was authenticated, not that GPU telemetry +// is valid. Any transport, decoding, or identity failure returns false and an +// empty response so stale or unverified measurements cannot survive the probe. +func (m *Manager) probeNodeInfo(client *http.Client, addr string, port int) (bool, NodeInfoResponse) { + url := "https://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/node-info" start := time.Now() - resp, err := client.Get(url) + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return false, NodeInfoResponse{} + } + // Resolve manual hostnames again on every probe, just as engine reads do, + // so a reassigned name cannot retain the previous host's hardware identity. + request.Close = true + resp, err := client.Do(request) if err != nil { slog.Debug("manual probe node-info failed", - "addr", addr, "port", port, "scheme", scheme, + "addr", addr, "port", port, "duration_ms", time.Since(start).Milliseconds(), "err", err) return false, NodeInfoResponse{} } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { slog.Debug("manual probe node-info non-OK", - "addr", addr, "port", port, "scheme", scheme, "status", resp.StatusCode, + "addr", addr, "port", port, "status", resp.StatusCode, "duration_ms", time.Since(start).Milliseconds()) return false, NodeInfoResponse{} } var result NodeInfoResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + body, err := io.ReadAll(io.LimitReader(resp.Body, (64<<10)+1)) + if err != nil || len(body) > 64<<10 { + // Partial or oversized data cannot supply a complete accepted snapshot. + return false, NodeInfoResponse{} + } + if err := json.Unmarshal(body, &result); err != nil { slog.Debug("manual probe node-info decode failed", - "addr", addr, "port", port, "scheme", scheme, "err", err, + "addr", addr, "port", port, "err", err, "duration_ms", time.Since(start).Milliseconds()) return false, NodeInfoResponse{} } + m.mesh.Refresh() + if resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { + // Without the server's TLS certificate, the response has no identity + // proof to check against our pins. + return false, NodeInfoResponse{} + } + if result.HostUUID == "" || result.HostUUID == m.mesh.NodeUUID() { + // The snapshot must name a remote host. Our own measurements come from + // the locally owned worker, never from a manually entered HTTP endpoint. + return false, NodeInfoResponse{} + } + if result.ClusterUUID == nil || *result.ClusterUUID != result.HostUUID { + // The response's principal must name the same host as its hardware data. + // A missing or conflicting principal cannot authorize that snapshot. + return false, NodeInfoResponse{} + } + if !m.mesh.VerifyPeerCertificate(result.HostUUID, resp.TLS.PeerCertificates[0]) { + // Recheck the actual server certificate after reading the body: its pin + // may have been removed or replaced since the TLS handshake completed. + return false, NodeInfoResponse{} + } + aged := (noderec.NodeInfo{ + TelemetryValid: result.TelemetryValid, + MSSince: result.MSSince, + }).Aged(time.Since(start)) + result.MSSince = aged.MSSince slog.Debug("manual probe node-info up", - "addr", addr, "port", port, "scheme", scheme, "gpus", len(result.GPUs), + "addr", addr, "port", port, "gpus", len(result.GPUs), "has_cpu", result.CPU != nil, "has_memory", result.Memory != nil, "duration_ms", time.Since(start).Milliseconds()) return true, result @@ -536,13 +594,14 @@ func (m *Manager) addNode(entry ManualEntry) ManualNodeStatus { nodeInfoPort = entry.TLSPort } status := ManualNodeStatus{ - ID: id, - Name: entry.Name, - Address: entry.Address, - OllamaPort: 11434, - NodeInfoPort: nodeInfoPort, - TLSEnabled: entry.TLSPort > 0, - MTLSRequired: entry.TLSPort > 0 && entry.MTLS, + IdentityVerification: noderec.IdentityUnverified, + ID: id, + Name: entry.Name, + Address: entry.Address, + OllamaPort: 11434, + NodeInfoPort: nodeInfoPort, + TLSEnabled: entry.TLSPort > 0, + MTLSRequired: entry.TLSPort > 0 && entry.MTLS, } m.mu.Lock() @@ -712,47 +771,3 @@ func sliceEqual(a, b []string) bool { } return true } - -func gpusEqual(a, b []GPUInfo) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i].Name != b[i].Name || - a[i].VramBytes != b[i].VramBytes || - a[i].VramUsedBytes != b[i].VramUsedBytes || - a[i].UtilizationPercent != b[i].UtilizationPercent { - return false - } - } - return true -} - -// cpuEqual and memoryEqual are nil-aware equality helpers for the new -// pointer-optional fields. Both sides being nil is equal (the node -// doesn't report that subsystem); exactly one being nil is a real -// change (the node started or stopped reporting it, e.g. an OS-level -// upgrade of nvpair-node-info); both non-nil compares the underlying -// struct by value. This mirrors what sliceEqual / gpusEqual do for -// their respective fields — any change, including a utilization tick -// moving from 41% to 42%, fires a node/updated event so downstream -// consumers can refresh their view. -func cpuEqual(a, b *CPUInfo) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - return *a == *b -} - -func memoryEqual(a, b *MemoryInfo) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - return *a == *b -} diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index 30c20b569..496d1fe62 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -11,6 +11,10 @@ import ( "io" "net" "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" "strings" "sync" "testing" @@ -20,6 +24,9 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/applog" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" ) type captureRW struct { @@ -76,16 +83,42 @@ func httpJSON(status int, body string) (*http.Response, error) { func newTestManager() (*Manager, *captureRW, *fakeRoundTripper) { rw := newCaptureRW() - m, err := NewManager(NewCodec(rw), tlsClientOptions{}, nil) - if err != nil { - panic(err) - } + m := NewManager(NewCodec(rw), nil) rt := newFakeRoundTripper() m.client = &http.Client{Transport: rt, Timeout: time.Second} - m.tlsClient = &http.Client{Transport: rt, Timeout: time.Second} return m, rw, rt } +// Configure a reachable engine and public hardware endpoint without cluster +// pins. The engine must remain usable, but the worker must never fetch or accept +// the public hardware claim as verified identity or scheduling measurements. +func TestProbeNode_PublicHardwareCannotSupplyServiceMeasurements(t *testing.T) { + const address = "192.0.2.20" + const modelName = "model-a" + m, _, transport := newTestManager() + entry := ManualEntry{Address: address, Name: "manual"} + configureHealthyNode(t, transport, entry.Address, []string{modelName}, NodeInfoResponse{ + HostUUID: "victim", + TelemetryValid: true, + GPUs: []GPUInfo{{UtilizationPercent: 1}}, + }) + var hardwareReads int + transport.set(http.MethodGet, address+":14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { + hardwareReads++ + return httpJSON(http.StatusOK, `{"hostUuid":"victim","telemetryValid":true}`) + }) + m.nodes[nodeID(entry)] = &trackedNode{entry: entry} + + m.probeNode(entry) + + status := m.nodes[nodeID(entry)].status + assert.Zero(t, hardwareReads) + assert.Empty(t, status.HostUUID) + assert.False(t, status.TelemetryValid) + assert.True(t, status.OllamaUp) + assert.Equal(t, []string{modelName}, status.OllamaModels) +} + func configureHealthyNode(t *testing.T, rt *fakeRoundTripper, addr string, models []string, info NodeInfoResponse) { t.Helper() host := net.JoinHostPort(addr, "11434") @@ -280,13 +313,19 @@ func TestNodeID_PrefersConfiguredNameOverAddressFallback(t *testing.T) { assert.Equal(t, "manual:10.0.0.5", nodeID(ManualEntry{Address: "10.0.0.5"}), "unnamed nodeID") } -func TestNodeAdd_RespondsWithInitialStatusThenDiscoversProbeResult(t *testing.T) { +func TestNodeAdd_RespondsWithInitialStatusAndDiscoversProbeResult(t *testing.T) { m, rw, rt := newTestManager() configureHealthyNode(t, rt, "node.local", []string{"llama3", "mistral"}, sampleInfo()) m.handleMessage(requestMessage(t, 1, "node/add", ManualEntry{Address: "node.local", Name: "lab"})) - resp := readCaptureUntil(t, rw, responseWithID(t, 1)) + var discovered Message + resp := readCaptureUntil(t, rw, func(msg Message) bool { + if msg.Method == "node/discovered" { + discovered = msg + } + return responseWithID(t, 1)(msg) + }) initial := decodeResult[ManualNodeStatus](t, resp) assert.Equal(t, "lab", initial.ID, "initial status (%v)", initial) assert.Equal(t, "node.local", initial.Address, "initial status (%v)", initial) @@ -295,19 +334,15 @@ func TestNodeAdd_RespondsWithInitialStatusThenDiscoversProbeResult(t *testing.T) assert.False(t, initial.OllamaUp, "initial status should be unprobed (%v)", initial) assert.False(t, initial.NodeInfoUp, "initial status should be unprobed (%v)", initial) - discovered := readCaptureUntil(t, rw, methodIs("node/discovered")) + if discovered.Method == "" { + discovered = readCaptureUntil(t, rw, methodIs("node/discovered")) + } status := decodeParams[ManualNodeStatus](t, discovered) assert.True(t, status.OllamaUp, "discovered status did not include healthy services (%v)", status) - assert.True(t, status.NodeInfoUp, "discovered status did not include healthy services (%v)", status) + assert.False(t, status.NodeInfoUp) assert.Equal(t, []string{"llama3", "mistral"}, status.OllamaModels) - require.Len(t, status.GPUs, 1) - assert.Equal(t, "RTX 6000", status.GPUs[0].Name) - require.NotNil(t, status.CPU) - assert.Equal(t, uint32(64), status.CPU.Cores) - require.NotNil(t, status.Memory) - assert.Equal(t, uint64(32<<30), status.Memory.UsedBytes) - assert.True(t, status.TelemetryValid, "telemetry = valid") - assert.Equal(t, int64(137), status.MSSince, "telemetry = valid") + assert.Empty(t, status.GPUs) + assert.False(t, status.TelemetryValid) } func TestNodeAdd_RejectsInvalidParameters(t *testing.T) { @@ -342,7 +377,7 @@ func TestNodesList_ReturnsCurrentStatuses(t *testing.T) { require.NoError(t, json.Unmarshal(resp.Result, &result), "decode nodes/list") require.Len(t, result.Nodes, 1, "nodes/list length") assert.True(t, result.Nodes[0].OllamaUp, "nodes/list status") - assert.True(t, result.Nodes[0].NodeInfoUp, "nodes/list status") + assert.False(t, result.Nodes[0].NodeInfoUp) } func TestNodeRemove_ReturnsRemovedAndNotifies(t *testing.T) { @@ -430,83 +465,306 @@ func TestProbeNode_EmitsNothingWhenStateIsUnchanged(t *testing.T) { assertNoCaptureMethod(t, rw, "node/updated") } -func TestProbeNode_ClearsAvailabilityAfterFailure(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} - m.nodes["lab"] = &trackedNode{ - entry: entry, - status: ManualNodeStatus{ - ID: "lab", - Address: "node.local", - OllamaUp: true, - OllamaPort: 11434, - OllamaModels: []string{"llama3"}, - NodeInfoUp: true, - NodeInfoPort: 14318, - GPUs: sampleInfo().GPUs, - CPU: sampleInfo().CPU, - Memory: sampleInfo().Memory, - }, +func TestProbeNode_DiscardsInvalidAuthenticatedHardware(t *testing.T) { + test := func(name string, respond func(*testing.T, http.ResponseWriter, []byte)) { + t.Run(name, func(t *testing.T) { + f := newManualPeerFixture(t) + engineTransport, ok := f.manager.client.Transport.(*fakeRoundTripper) + require.True(t, ok) + configureHealthyLMStudio(t, engineTransport, f.entry.Address, []string{"model-a"}) + f.manager.probeNode(f.entry) + initial := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, f.frames, methodIs("node/updated"))) + require.Equal(t, noderec.IdentityPinned, initial.IdentityVerification) + require.True(t, initial.NodeInfoUp) + require.NotEmpty(t, initial.GPUs) + require.NotNil(t, initial.CPU) + require.NotNil(t, initial.Memory) + body, err := json.Marshal(f.info) + require.NoError(t, err) + requested := make(chan struct{}, 1) + f.respond = func(w http.ResponseWriter) { + requested <- struct{}{} + respond(t, w, body) + } + + f.manager.probeNode(f.entry) + + select { + case <-requested: + default: + t.Fatal("the authenticated hardware request did not reach the peer") + } + updated := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, f.frames, methodIs("node/updated"))) + assert.Equal(t, noderec.IdentityUnverified, updated.IdentityVerification) + assert.Empty(t, updated.HostUUID) + assert.False(t, updated.NodeInfoUp) + assert.Empty(t, updated.GPUs) + assert.Nil(t, updated.CPU) + assert.Nil(t, updated.Memory) + assert.False(t, updated.TelemetryValid) + assert.Zero(t, updated.MSSince) + assert.True(t, updated.LMStudioUp) + assert.Equal(t, []string{"model-a"}, updated.LMStudioModels) + }) } - rt.set(http.MethodGet, net.JoinHostPort("node.local", "11434"), "/", func(*http.Request) (*http.Response, error) { - return nil, errors.New("ollama down") + test("malformed JSON", func(t *testing.T, w http.ResponseWriter, _ []byte) { + _, err := w.Write([]byte(`{not json`)) + assert.NoError(t, err) }) - rt.set(http.MethodGet, net.JoinHostPort("node.local", "14318"), "/v1/node-info", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{not json`) + test("truncated response with valid JSON", func(t *testing.T, w http.ResponseWriter, body []byte) { + // The JSON is complete, but fewer bytes than promised must still fail. + w.Header().Set("Content-Length", strconv.Itoa(len(body)+1)) + _, err := w.Write(body) + assert.NoError(t, err) }) + test("response larger than 64 KiB", func(t *testing.T, w http.ResponseWriter, body []byte) { + // Valid trailing whitespace isolates the size limit from JSON parsing. + oversized := append(body, []byte(strings.Repeat(" ", (64<<10)+1-len(body)))...) + _, err := w.Write(oversized) + assert.NoError(t, err) + }) +} - m.probeNode(entry) - updated := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) - assert.False(t, updated.OllamaUp, "services should be down (%v)", updated) - assert.False(t, updated.NodeInfoUp, "services should be down (%v)", updated) - require.Empty(t, updated.OllamaModels, "failed probe retained stale fields (%v)", updated) - require.Empty(t, updated.GPUs, "failed probe retained stale fields (%v)", updated) - require.Nil(t, updated.CPU, "failed probe retained stale fields (%v)", updated) - require.Nil(t, updated.Memory, "failed probe retained stale fields (%v)", updated) -} - -// TestProbeFailurePreservesHostUUID: a node-info blip -// (while Ollama stays reachable) must NOT blank the learned HostUUID, or the -// broker would rekey the live node UUID -> manual-id -> UUID. -func TestProbeFailure_PreservesHostUUID(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} - m.nodes["lab"] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}} +const manualPeerUUID = "peer" - info := sampleInfo() - info.HostUUID = "node-uuid" - configureHealthyNode(t, rt, "node.local", []string{"llama3"}, info) +type manualPeerFixture struct { + manager *Manager + peerDir string + frames *captureRW + entry ManualEntry + info NodeInfoResponse + server *httptest.Server + respond func(http.ResponseWriter) +} - // First probe learns the UUID. - m.probeNode(entry) - first := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) - assert.Equal(t, "node-uuid", first.HostUUID, "first probe HostUUID") +// Create isolated cluster identities pinned in both directions and a local +// mTLS node-info server. The manager uses that real authenticated connection +// while engine probes remain fake, so the fixture never needs a remote host. +func newManualPeerFixture(t *testing.T) *manualPeerFixture { + t.Helper() + const clusterID = "test-cluster" + const selfUUID = "self" + clientDir, serverDir := t.TempDir(), t.TempDir() + clustertrusttest.Join(t, clientDir, clusterID, selfUUID) + clustertrusttest.Join(t, serverDir, clusterID, manualPeerUUID) + pin := func(destination, source, uuid string) { + t.Helper() + certificate, err := os.ReadFile(filepath.Join(source, "node.crt")) + require.NoError(t, err) + type peerPin struct { + NodeUUID string `json:"nodeUuid"` + CertPEM string `json:"certPem"` + } + data, err := json.Marshal(peerPin{NodeUUID: uuid, CertPEM: string(certificate)}) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(destination, "trusted"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(destination, "trusted", uuid+".json"), data, 0o600)) + } + pin(clientDir, serverDir, manualPeerUUID) + pin(serverDir, clientDir, selfUUID) + manager, frames, _ := newTestManager() + manager.mesh = clustertrust.Open(clientDir) + manager.peerClients = clustertrust.NewPeerClientPool(manager.mesh, probeTimeout) + t.Cleanup(manager.peerClients.CloseIdle) + f := &manualPeerFixture{ + manager: manager, + peerDir: serverDir, + frames: frames, + info: sampleInfo(), + } + f.info.HostUUID = manualPeerUUID + principal := manualPeerUUID + f.info.ClusterUUID = &principal + peerMesh := clustertrust.Open(serverDir) + f.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := peerMesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + if f.respond != nil { + f.respond(w) + return + } + assert.NoError(t, json.NewEncoder(w).Encode(f.info)) + })) + f.server.TLS = peerMesh.ServerTLSConfig() + f.server.StartTLS() + t.Cleanup(f.server.Close) + host, port, err := net.SplitHostPort(f.server.Listener.Addr().String()) + require.NoError(t, err) + portNumber, err := strconv.Atoi(port) + require.NoError(t, err) + f.entry = ManualEntry{ + Address: host, + Name: "manual-peer", + TLSPort: portNumber, + } + manager.nodes[nodeID(f.entry)] = &trackedNode{entry: f.entry} + return f +} + +// Probe a mutually pinned peer and check that its hardware and host identity +// enter the manual status together, with sample age retained across the fetch. +func TestProbeNode_AcceptsMeasurementsFromPinnedPeer(t *testing.T) { + f := newManualPeerFixture(t) + + f.manager.probeNode(f.entry) + + status := f.manager.listNodes() + require.Len(t, status, 1) + assert.Equal(t, noderec.IdentityPinned, status[0].IdentityVerification) + assert.Equal(t, manualPeerUUID, status[0].HostUUID) + assert.Equal(t, f.info.GPUs, status[0].GPUs) + assert.True(t, status[0].TelemetryValid) + assert.GreaterOrEqual(t, status[0].MSSince, int64(137)) +} + +func TestProbeNode_DiscardsOldIdentityAfterHostnameReassignment(t *testing.T) { + f := newManualPeerFixture(t) + f.manager.probeNode(f.entry) + initial := f.manager.listNodes() + require.Len(t, initial, 1) + require.Equal(t, noderec.IdentityPinned, initial[0].IdentityVerification) + + client, ok := f.manager.peerClients.Client(manualPeerUUID) + require.True(t, ok) + transport, ok := client.Transport.(*http.Transport) + require.True(t, ok) + engineTransport, ok := f.manager.client.Transport.(*fakeRoundTripper) + require.True(t, ok) + + // Resolve the manual hostname inside the dialer so reassignment is local + // and deterministic. The first server stays alive throughout the test. + var targetMu sync.Mutex + target := f.server.Listener.Addr().String() + var dialed []string + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + targetMu.Lock() + address := target + dialed = append(dialed, address) + targetMu.Unlock() + return (&net.Dialer{}).DialContext(ctx, network, address) + } + f.entry.Address = "manual.example" + f.manager.nodes[nodeID(f.entry)].entry = f.entry + configureHealthyNode(t, engineTransport, f.entry.Address, []string{"original-model"}, NodeInfoResponse{}) + f.manager.probeNode(f.entry) + warmed := f.manager.listNodes() + require.Len(t, warmed, 1) + require.Equal(t, manualPeerUUID, warmed[0].HostUUID) + require.True(t, warmed[0].NodeInfoUp) + + // The replacement has a different certificate. A fresh connection must + // reject it rather than keep attributing its engine to the original host. + replacement := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(replacement.Close) + targetMu.Lock() + target = replacement.Listener.Addr().String() + targetMu.Unlock() + configureHealthyNode(t, engineTransport, f.entry.Address, []string{"replacement-model"}, NodeInfoResponse{}) + + f.manager.probeNode(f.entry) + + status := f.manager.listNodes() + require.Len(t, status, 1) + assert.Equal(t, []string{"replacement-model"}, status[0].OllamaModels) + assert.Equal(t, noderec.IdentityUnverified, status[0].IdentityVerification) + assert.False(t, status[0].NodeInfoUp) + assert.Empty(t, status[0].HostUUID) + assert.Empty(t, status[0].GPUs) + assert.False(t, status[0].TelemetryValid) + targetMu.Lock() + assert.Equal(t, []string{f.server.Listener.Addr().String(), replacement.Listener.Addr().String()}, dialed) + targetMu.Unlock() +} + +// Make the manual endpoint present our own certificate. Even a successful TLS +// connection must not let network measurements impersonate the owned local +// worker or grant this manual entry verified local identity. +func TestProbeNode_LocalIdentityCannotSupplyMeasurementsOverHTTP(t *testing.T) { + f := newManualPeerFixture(t) + // The entered endpoint now presents our own identity. Local measurements + // must come from the owned worker, even when this TLS handshake succeeds. + f.manager.mesh = clustertrust.Open(f.peerDir) + f.manager.peerClients = clustertrust.NewPeerClientPool(f.manager.mesh, probeTimeout) + t.Cleanup(f.manager.peerClients.CloseIdle) + + f.manager.probeNode(f.entry) + + status := f.manager.listNodes() + require.Len(t, status, 1) + assert.Equal(t, noderec.IdentityUnverified, status[0].IdentityVerification) + assert.False(t, status[0].TelemetryValid) + assert.Empty(t, status[0].HostUUID) +} + +// Repeated authenticated reads must each publish a snapshot, even when hardware +// values stay identical, so downstream caches continue to observe live samples. +func TestProbeNode_PublishesUnchangedAuthenticatedMeasurements(t *testing.T) { + const updatedMethod = "node/updated" + f := newManualPeerFixture(t) + // Invalid measurements always have age zero, so neither age nor inventory + // changes can accidentally trigger this periodic notification. + f.info.TelemetryValid, f.info.MSSince = false, 0 + f.manager.probeNode(f.entry) + readCaptureUntil(t, f.frames, methodIs(updatedMethod)) + + f.manager.probeNode(f.entry) + + message := readCaptureUntil(t, f.frames, methodIs(updatedMethod)) + status := decodeParams[ManualNodeStatus](t, message) + assert.Equal(t, noderec.IdentityPinned, status.IdentityVerification) + assert.False(t, status.TelemetryValid) +} + +// Keep a valid pinned TLS connection but vary the response's host and principal +// claims. A conflicting or missing identity must leave the manual entry +// unverified and discard its hardware, despite the successful TLS handshake. +func TestProbeNode_RejectsConflictingPinnedResponseIdentity(t *testing.T) { + test := func(name, host, principal string) { + t.Run(name, func(t *testing.T) { + f := newManualPeerFixture(t) + f.info.HostUUID, f.info.ClusterUUID = host, &principal - // node-info goes down while Ollama stays up: the UUID must be preserved. - rt.set(http.MethodGet, net.JoinHostPort("node.local", "14318"), "/v1/node-info", func(*http.Request) (*http.Response, error) { - return nil, errors.New("node-info down") - }) - m.probeNode(entry) - down := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) - assert.False(t, down.NodeInfoUp, "node-info should be down") - assert.Equal(t, "node-uuid", down.HostUUID, "HostUUID dropped on node-info failure") + f.manager.probeNode(f.entry) - // node-info recovers: still the same UUID (no flap). - configureHealthyNode(t, rt, "node.local", []string{"llama3"}, info) - m.probeNode(entry) - up := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) - assert.Equal(t, "node-uuid", up.HostUUID, "HostUUID after recovery") + status := f.manager.listNodes() + require.Len(t, status, 1) + assert.Equal(t, noderec.IdentityUnverified, status[0].IdentityVerification) + assert.Empty(t, status[0].HostUUID) + assert.False(t, status[0].TelemetryValid) + }) + } + test("another host", "different-host", manualPeerUUID) + test("another principal", manualPeerUUID, "different-principal") + test("missing principal", manualPeerUUID, "") } -func TestTelemetryEquality_DistinguishesMissingCPUAndMemoryReadings(t *testing.T) { - assert.True(t, cpuEqual(nil, nil), "nil values should compare equal") - assert.True(t, memoryEqual(nil, nil), "nil values should compare equal") - assert.False(t, cpuEqual(nil, &CPUInfo{}), "nil and non-nil values should differ") - assert.False(t, memoryEqual(nil, &MemoryInfo{}), "nil and non-nil values should differ") - assert.True(t, cpuEqual(&CPUInfo{Name: "cpu", Cores: 8}, &CPUInfo{Name: "cpu", Cores: 8}), "equal CPU values differed") - assert.False(t, cpuEqual(&CPUInfo{Name: "cpu", Cores: 8}, &CPUInfo{Name: "cpu", Cores: 16}), "different CPU values compared equal") - assert.True(t, memoryEqual(&MemoryInfo{TotalBytes: 10, UsedBytes: 5}, &MemoryInfo{TotalBytes: 10, UsedBytes: 5}), "equal memory values differed") - assert.False(t, memoryEqual(&MemoryInfo{TotalBytes: 10, UsedBytes: 5}, &MemoryInfo{TotalBytes: 10, UsedBytes: 6}), "different memory values compared equal") +func TestProbeNode_PublishesCPUAndMemoryAvailabilityChanges(t *testing.T) { + test := func(name string, beforeCPU, afterCPU *CPUInfo, beforeMemory, afterMemory *MemoryInfo) { + t.Run(name, func(t *testing.T) { + f := newManualPeerFixture(t) + f.info.CPU, f.info.Memory = beforeCPU, beforeMemory + f.info.TelemetryValid, f.info.MSSince = false, 0 + f.manager.probeNode(f.entry) + readCaptureUntil(t, f.frames, methodIs("node/updated")) + + f.info.CPU, f.info.Memory = afterCPU, afterMemory + f.manager.probeNode(f.entry) + + updated := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, f.frames, methodIs("node/updated"))) + assert.Equal(t, noderec.IdentityPinned, updated.IdentityVerification) + assert.True(t, updated.NodeInfoUp) + assert.Equal(t, afterCPU, updated.CPU) + assert.Equal(t, afterMemory, updated.Memory) + }) + } + test("CPU becomes available", nil, &CPUInfo{Name: "cpu", Cores: 8}, nil, nil) + test("CPU becomes unavailable", &CPUInfo{Name: "cpu", Cores: 8}, nil, nil, nil) + test("memory becomes available", nil, nil, nil, &MemoryInfo{TotalBytes: 10, UsedBytes: 5}) + test("memory becomes unavailable", nil, nil, &MemoryInfo{TotalBytes: 10, UsedBytes: 5}, nil) } func TestUnknownMethod_ReturnsMethodNotFound(t *testing.T) { @@ -535,8 +793,7 @@ func TestShutdownRequest_CancelsRun(t *testing.T) { defer server.Close() defer client.Close() - mgr, err := NewManager(NewCodec(server), tlsClientOptions{}, nil) - require.NoError(t, err, "NewManager") + mgr := NewManager(NewCodec(server), nil) done := make(chan error, 1) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/services/nvpair-manual-nodes/spec.md b/services/nvpair-manual-nodes/spec.md index 0d1508783..c68296090 100644 --- a/services/nvpair-manual-nodes/spec.md +++ b/services/nvpair-manual-nodes/spec.md @@ -48,7 +48,7 @@ It exists because `nvpair-node-scanner` has no manual concept at all: the scanne ## 4. Open Questions / Risks - **Worker restart**: current probe results are lost. The broker withdraws the retired worker's records and samples, then replays durable entries to the replacement worker. Restored configurations start unverified and must authenticate again before supplying hardware data. -- **Risk — `node/updated` on every probe**: change detection compares `cpu`, `memory`, and `gpus` by value, and also `telemetryValid` and `msSince`. Because `msSince` is the remote GPU sample's age at response time, it differs on essentially every probe, so any node whose `nvpair-node-info` reports valid GPU telemetry emits a full `node/updated` every 10 s. Each one makes the broker re-project the discovery snapshot, re-bridge the node into `nvpair-proxy`, and re-ingest its telemetry. +- **Hardware refresh cost**: every accepted authenticated hardware snapshot emits a full `node/updated`, even when measurements are unchanged, so scheduler caches keep observing live samples. Each one makes the broker re-project the discovery snapshot, re-bridge the node into `nvpair-proxy`, and re-ingest its telemetry. Change detection covers configuration, identity, engine models, and service availability rather than hardware values. - **llama.cpp is not supported (open — gap)**: llama.cpp is the third registered engine, but the prober checks only `11434` and `1234`, `ManualNodeStatus` has no llama.cpp fields, and the broker's `bridgeManualNode`, `removeManualNodeFromProxies`, and `manualModelsByEngine` handle only `ollama` and `lmstudio`. A manual-only node therefore never enters the llama.cpp facade's candidate set and advertises no llama.cpp models. A machine that mDNS also discovers still reaches that facade through the broker's discovery relay under its `lc` record. - **Risk — fixed engine and telemetry ports**: `11434` (Ollama), `1234` (LM Studio), and `14318` (pinned node-info) are compiled in. A remote engine on a non-default port is invisible, and an address entered as `host:port` produces a malformed probe URL, so the node reads permanently down. Only `tls_port` lets a caller move a probe port, and only for the node-info leg. - **Risk — non-EOF read errors loop**: `readLoop` treats every non-`io.EOF` read error as recoverable and continues. `nvpair-shared/jsonrpc` distinguishes a per-frame `DecodeError` from a terminal transport error, but the manager ignores that distinction, so a permanently failing pipe spins the loop logging on each iteration. diff --git a/services/nvpair-manual-nodes/tls.go b/services/nvpair-manual-nodes/tls.go deleted file mode 100644 index 6c983d9b3..000000000 --- a/services/nvpair-manual-nodes/tls.go +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "net/http" - "os" - "time" -) - -// tlsClientOptions captures the cert paths the manager uses when -// probing TLS-enabled manual nodes. All three are optional. When -// they're all empty, the manager still gets a working tlsClient -// (system trust store, no client cert) so a probe of a public-CA -// HTTPS node works without any flags. When --client-cert and -// --client-key are set they're sent on every TLS probe — there's -// no per-node "send cert here, don't there" — because the -// originating node is the user's own machine and the cost of -// always presenting a cert is zero. -type tlsClientOptions struct { - CertPath string - KeyPath string - CABundlePath string -} - -func (o tlsClientOptions) validate() error { - switch { - case o.CertPath != "" && o.KeyPath == "": - return errors.New("--client-cert requires --client-key") - case o.KeyPath != "" && o.CertPath == "": - return errors.New("--client-key requires --client-cert") - } - return nil -} - -// buildTLSClient mirrors the scanner's helper of the same name — -// the two would live in shared/applog if Go modules made local -// shared code less cumbersome to import, but for two ~30-line -// helpers it's cheaper to duplicate. Returns nil when nothing -// is configured; the caller falls back to a default *http.Client -// (system trust store) so out-of-the-box HTTPS to public-CA nodes -// just works. -func buildTLSClient(o tlsClientOptions, timeout time.Duration) (*http.Client, error) { - if o.CertPath == "" && o.KeyPath == "" && o.CABundlePath == "" { - return nil, nil - } - cfg := &tls.Config{ - MinVersion: tls.VersionTLS12, - } - if o.CABundlePath != "" { - caPEM, err := os.ReadFile(o.CABundlePath) - if err != nil { - return nil, fmt.Errorf("read CA bundle: %w", err) - } - pool, err := x509.SystemCertPool() - if err != nil || pool == nil { - pool = x509.NewCertPool() - } - if !pool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("CA bundle %s contains no PEM certificates", o.CABundlePath) - } - cfg.RootCAs = pool - } - if o.CertPath != "" { - cert, err := tls.LoadX509KeyPair(o.CertPath, o.KeyPath) - if err != nil { - return nil, fmt.Errorf("load client cert/key: %w", err) - } - cfg.Certificates = []tls.Certificate{cert} - } - return &http.Client{ - Timeout: timeout, - // DisableKeepAlives so each probe re-resolves the entry's address - // rather than reusing a pooled connection to a now-stale IP. - Transport: &http.Transport{TLSClientConfig: cfg, DisableKeepAlives: true}, - }, nil -} diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index fb7e4d5aa..50fd83c3e 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -679,6 +679,7 @@ func (d *daemon) publishPeerInfo(candidateID string, sample peerInfoSample) { } info = sample.info.Aged(time.Since(sample.receivedAt)) d.emitTelemetry(noderec.NodeTelemetry{HostUUID: info.HostUUID, GPUUtilizationPct: utilization, TelemetryValid: info.TelemetryValid, MSSince: info.MSSince}) + d.emitNodeInfo(node, sample.info.Aged(time.Since(sample.receivedAt))) } // onBrowse stores each discovery independently. Network claims never mutate a @@ -841,6 +842,18 @@ func (d *daemon) emitTelemetry(telemetry noderec.NodeTelemetry) { } } +func (d *daemon) emitNodeInfo(node noderec.DirectoryNode, info noderec.NodeInfo) { + if d.codec == nil { + return + } + if err := d.codec.Notify(noderec.NotifyNodeInfo, noderec.NodeInfoEvent{ + DiscoveryRecordID: node.DiscoveryRecordID, HostUUID: node.HostUUID, + ClusterUUID: node.ClusterUUID, IdentityVerification: node.IdentityVerification, Info: info, + }); err != nil { + slog.Warn("failed to emit node-info", "host_uuid", node.HostUUID, "err", err) + } +} + // reloadIdentity re-resolves this node's advertised identity (hostUuid, // clusterUuid, ip) and re-advertises if any of it changed. It has three triggers: // @@ -1012,7 +1025,7 @@ func (d *daemon) dropSelf(oldHostUUID, newHostUUID string) { delete(d.lastLoadedByEngine, oldHostUUID) d.infoMu.Unlock() if existed { - d.emit(noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: oldHostUUID, Name: node.Name}) + d.emit(noderec.NotifyNodeRemoved, node) } } @@ -1119,11 +1132,14 @@ func (d *daemon) acceptLocalNodeInfo(info noderec.NodeInfo) { utilization = gpu.UtilizationPercent } } - info = info.Aged(time.Since(receivedAt)) + aged := info.Aged(time.Since(receivedAt)) d.emitTelemetry(noderec.NodeTelemetry{ HostUUID: self, GPUUtilizationPct: utilization, - TelemetryValid: info.TelemetryValid, MSSince: info.MSSince, + TelemetryValid: aged.TelemetryValid, MSSince: aged.MSSince, }) + if current, ok := d.dir.get(self); ok { + d.emitNodeInfo(current, info.Aged(time.Since(receivedAt))) + } } // watchAddress re-derives the advertised ip= whenever the host's network changes diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index 1653262d5..468de7e13 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -328,6 +328,29 @@ func TestPublishSelf_LocalRecordHasLocalVerificationAndDiscoveryRecordID(t *test assert.Equal(t, "local:self-uuid", node.DiscoveryRecordID) } +// Replace the local identity and inspect the old record's removal event. Its +// local provenance and discovery record ID must survive so consumers invalidate +// the correct source instead of treating it as an unverified disappearance. +func TestDropSelf_PreservesRecordIdentityInRemoval(t *testing.T) { + const oldUUID = "old-self" + d := newSelfTestDaemon(oldUUID, "192.0.2.20") + d.publishSelf() + var output bytes.Buffer + d.codec = NewCodec(&output) + + d.selfMu.Lock() + d.dropSelf(oldUUID, "new-self") + d.selfMu.Unlock() + + message, err := NewCodec(&output).Read() + require.NoError(t, err) + var event noderec.NodeEvent + require.NoError(t, json.Unmarshal(message.Params, &event)) + assert.Equal(t, noderec.NotifyNodeRemoved, message.Method) + assert.Equal(t, "local:old-self", event.Node.DiscoveryRecordID) + assert.Equal(t, noderec.IdentityLocal, event.Node.IdentityVerification) +} + type discoveryPeerFixture struct { daemon *daemon selfDir string @@ -416,6 +439,81 @@ func TestDiscovery_ReplacesUnverifiedDiscoveryWithVerifiedPeer(t *testing.T) { assert.Len(t, f.daemon.dir.snapshot(""), 1) } +// Authenticate a real pinned test peer and inspect the scanner's outbound +// notifications. Exactly one full hardware snapshot must carry that verified +// record's identity and the hardware supplied by its authenticated response. +func TestDiscovery_PublishesAuthenticatedHardwareSnapshots(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + f.discover(t) + + codec := NewCodec(&output) + var snapshots int + for { + message, err := codec.Read() + if err == io.EOF { + break + } + require.NoError(t, err) + if message.Method != "discovery:node-info" { + continue + } + type hardwareSnapshot struct { + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + Info noderec.NodeInfo `json:"info"` + } + var snapshot hardwareSnapshot + require.NoError(t, json.Unmarshal(message.Params, &snapshot)) + assert.Equal(t, "peer:peer-uuid", snapshot.DiscoveryRecordID) + assert.Equal(t, noderec.IdentityPinned, snapshot.IdentityVerification) + assert.Equal(t, f.info.GPUs, snapshot.Info.GPUs) + snapshots++ + } + assert.Equal(t, 1, snapshots) +} + +type delayedNodeEventsWriter struct { + bytes.Buffer +} + +func (w *delayedNodeEventsWriter) Write(frame []byte) (int, error) { + var message Message + if err := json.Unmarshal(frame, &message); err != nil { + return 0, err + } + if message.Method == noderec.NotifyNodeDiscovered { + time.Sleep(100 * time.Millisecond) + } + return w.Buffer.Write(frame) +} + +// Delay the directory notification ahead of the hardware notification by +// 100 ms. The later snapshot must include that delay in its GPU sample age, +// preventing a blocked notification write from making old data look fresh. +func TestDiscovery_SnapshotAgeIncludesNotificationWait(t *testing.T) { + const peerUUID = "peer-uuid" + f := newDiscoveryPeerFixture(t, peerUUID) + output := &delayedNodeEventsWriter{} + f.daemon.codec = NewCodec(output) + + f.discover(t) + + decoder := json.NewDecoder(&output.Buffer) + var snapshot noderec.NodeInfoEvent + for decoder.More() { + var message Message + require.NoError(t, decoder.Decode(&message)) + if message.Method == noderec.NotifyNodeInfo { + require.NoError(t, json.Unmarshal(message.Params, &snapshot)) + } + } + assert.Equal(t, peerUUID, snapshot.HostUUID) + assert.GreaterOrEqual(t, snapshot.Info.MSSince, int64(fakePeerSampleAgeMillis+100)) +} + func TestDiscovery_FetchesModelsAfterAuthentication(t *testing.T) { f := newDiscoveryPeerFixture(t, "peer-uuid") f.info.Peer.Services[noderec.ServiceEngineManager] = portFromURL(t, f.server.URL) @@ -929,8 +1027,7 @@ func TestDiscovery_SlowAuthenticationDoesNotBlockVerifiedPeerTelemetry(t *testin var output bytes.Buffer f.daemon.codec = NewCodec(&output) f.daemon.refreshTelemetryOnce(context.Background()) - var message Message - require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message)) + message := readTelemetryMessage(t, json.NewDecoder(&output)) var sample noderec.NodeTelemetry require.NoError(t, json.Unmarshal(message.Params, &sample)) assert.Equal(t, "peer-uuid", sample.HostUUID) @@ -948,9 +1045,7 @@ func TestDiscovery_TelemetryIncludesSampleAgeAndReusesConnection(t *testing.T) { } decoder := json.NewDecoder(&output) for range 3 { - var message Message - require.NoError(t, decoder.Decode(&message)) - require.Equal(t, noderec.NotifyNodeTelemetry, message.Method) + message := readTelemetryMessage(t, decoder) var sample noderec.NodeTelemetry require.NoError(t, json.Unmarshal(message.Params, &sample)) assert.Equal(t, "peer-uuid", sample.HostUUID) @@ -989,8 +1084,7 @@ func TestDiscovery_InvalidTelemetryStaysInvalidAndReportsNoAge(t *testing.T) { var output bytes.Buffer f.daemon.codec = NewCodec(&output) f.daemon.refreshTelemetryOnce(context.Background()) - var message Message - require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message)) + message := readTelemetryMessage(t, json.NewDecoder(&output)) var sample noderec.NodeTelemetry require.NoError(t, json.Unmarshal(message.Params, &sample)) assert.False(t, sample.TelemetryValid) diff --git a/services/nvpair-node-scanner/telemetry_test.go b/services/nvpair-node-scanner/telemetry_test.go index 0866abb4c..2b432ba5b 100644 --- a/services/nvpair-node-scanner/telemetry_test.go +++ b/services/nvpair-node-scanner/telemetry_test.go @@ -65,6 +65,7 @@ func TestTelemetryDispatch_UpdatesInventoryAndSchedulingTogether(t *testing.T) { assert.Equal(t, f.info.CPU, node.CPU) assert.Equal(t, f.info.GPUs, node.GPUs) var telemetry noderec.NodeTelemetry + var snapshot noderec.NodeInfoEvent decoder := json.NewDecoder(&output) for decoder.More() { var message Message @@ -72,11 +73,15 @@ func TestTelemetryDispatch_UpdatesInventoryAndSchedulingTogether(t *testing.T) { switch message.Method { case noderec.NotifyNodeTelemetry: require.NoError(t, json.Unmarshal(message.Params, &telemetry)) + case noderec.NotifyNodeInfo: + require.NoError(t, json.Unmarshal(message.Params, &snapshot)) } } assert.Equal(t, "peer-uuid", telemetry.HostUUID) assert.True(t, telemetry.TelemetryValid) assert.Equal(t, uint32(95), telemetry.GPUUtilizationPct) + assert.Equal(t, f.info.CPU, snapshot.Info.CPU) + assert.Equal(t, f.info.GPUs, snapshot.Info.GPUs) } func TestTelemetryDispatch_BackoffDoesNotInterruptHealthyPeer(t *testing.T) { diff --git a/services/nvpair-proxy/proxy.go b/services/nvpair-proxy/proxy.go index 6a9b157ed..9b337ca2b 100644 --- a/services/nvpair-proxy/proxy.go +++ b/services/nvpair-proxy/proxy.go @@ -2530,12 +2530,21 @@ func (p *Proxy) setPriorityLocked(priority schedulerwire.Priority) int { return len(cleaned) } -// subscribedToNode projects a relay DirectoryNode onto the proxy's routable Node -// for the ol service, returning false when the node doesn't advertise ol or has -// no dialable address. The engine port comes from the ol service key (the real -// Ollama port the broker's engine poller registered, not the proxy's listen -// port). +// subscribedToNode projects a local record or a trusted pinned peer into a +// routable node for this engine. It requires the engine's advertised service +// and a dialable IP. The service supplies the engine's registered server port. func subscribedToNode(p engineProfile, n noderec.DirectoryNode) (Node, bool) { + verification := n.IdentityVerification + if verification != noderec.IdentityLocal && verification != noderec.IdentityPinned { + // UUID, trusted, and loopback claims cannot make an unverified + // advertisement eligible for inference routing. + return Node{}, false + } + if verification == noderec.IdentityPinned && !n.Trusted { + // A proved remote identity still needs current trust. Local records + // come from the owned worker and do not require a peer pin. + return Node{}, false + } svc, ok := n.Services[p.DiscoveryService] if !ok || n.IP == "" { return Node{}, false diff --git a/services/nvpair-proxy/proxy_test.go b/services/nvpair-proxy/proxy_test.go index eea7fa645..916895970 100644 --- a/services/nvpair-proxy/proxy_test.go +++ b/services/nvpair-proxy/proxy_test.go @@ -8,8 +8,78 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "nvpair-shared/noderec" ) +// An unverified discovery can claim the local UUID, loopback address, and trust. +// None of those claims may bypass verification to create an inference target. +func TestSubscribedToNode_RejectsUnverifiedLoopbackClaims(t *testing.T) { + node := noderec.DirectoryNode{ + HostUUID: "self", + IP: "127.0.0.1", + Trusted: true, + IdentityVerification: noderec.IdentityUnverified, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceOllama: {Port: 11434}, + }, + } + profile, ok := profileFor("ollama") + require.True(t, ok) + _, accepted := subscribedToNode(profile, node) + assert.False(t, accepted) +} + +// Otherwise routable peer records must fail independently when verification +// is absent or unverified, or when a proved peer no longer has current trust. +func TestSubscribedToNode_RejectsUnverifiedOrUntrustedPeers(t *testing.T) { + profile, ok := profileFor("ollama") + require.True(t, ok) + test := func(name string, verification noderec.IdentityVerification, trusted bool) { + t.Run(name, func(t *testing.T) { + node := noderec.DirectoryNode{ + HostUUID: "peer", + IP: "192.0.2.20", + IdentityVerification: verification, + Trusted: trusted, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceOllama: {Port: 11434}, + }, + } + + _, accepted := subscribedToNode(profile, node) + + assert.False(t, accepted) + }) + } + test("unverified identity with claimed trust", noderec.IdentityUnverified, true) + test("missing verification with claimed trust", "", true) + test("pinned identity without current trust", noderec.IdentityPinned, false) +} + +// The owned local record is eligible before joining a cluster, so it does not +// need a remote peer pin or the corresponding trusted flag to remain routable. +func TestSubscribedToNode_AcceptsLocalRecordsWithoutPeerTrust(t *testing.T) { + const hostUUID = "self" + const enginePort = 11434 + profile, ok := profileFor("ollama") + require.True(t, ok) + node := noderec.DirectoryNode{ + HostUUID: hostUUID, + IP: "127.0.0.1", + IdentityVerification: noderec.IdentityLocal, + Trusted: false, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceOllama: {Port: enginePort}, + }, + } + + projected, accepted := subscribedToNode(profile, node) + + require.True(t, accepted) + assert.Equal(t, hostUUID, projected.ID) + assert.Equal(t, enginePort, projected.Port) +} + // TestNodeCandidates covers the deterministic, loopback-first ordering and // de-duplication of a node's advertised addresses. Local-address rewriting is // not exercised (it depends on the host's interfaces); these addresses are diff --git a/services/nvpair-proxy/subscribed_test.go b/services/nvpair-proxy/subscribed_test.go index 1502ea3fb..c16e642ca 100644 --- a/services/nvpair-proxy/subscribed_test.go +++ b/services/nvpair-proxy/subscribed_test.go @@ -95,23 +95,30 @@ func TestSubscribedDiff_ReportsAddedChangedAndRemovedNodes(t *testing.T) { // regresses if the projection ever loses its service-key filter. func TestSubscribedToNode_ProjectsDiscoveryFields(t *testing.T) { forEachEngine(t, func(t *testing.T, tc engineCase) { + const hostUUID = "uuid-a" + const address = "10.0.0.5" + const modelName = "llama" + const ownModel = "mine" + const otherModel = "theirs" other := otherEngine(t, tc) withService := noderec.DirectoryNode{ - HostUUID: "uuid-a", - Name: "host-a", - IP: "10.0.0.5", - Models: []string{"llama"}, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: hostUUID, + Name: "host-a", + IP: address, + Models: []string{modelName}, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ tc.profile.DiscoveryService: {Port: tc.profile.FacadePort}, }, } got, ok := subscribedToNode(tc.profile, withService) require.True(t, ok, "node advertising this engine + IP should project") - require.Equal(t, "uuid-a", got.ID, "unexpected projection (%v)", got) + require.Equal(t, hostUUID, got.ID, "unexpected projection (%v)", got) require.Equal(t, tc.profile.FacadePort, got.Port, "unexpected projection (%v)", got) - require.Equal(t, "10.0.0.5", got.IP, "unexpected projection (%v)", got) - require.Equal(t, []string{"llama"}, got.Models, "unexpected projection (%v)", got) + require.Equal(t, address, got.IP, "unexpected projection (%v)", got) + require.Equal(t, []string{modelName}, got.Models, "unexpected projection (%v)", got) noIP := withService noIP.IP = "" @@ -121,35 +128,45 @@ func TestSubscribedToNode_ProjectsDiscoveryFields(t *testing.T) { // A node advertising only a non-engine service (node-info) is not a // routing target. niOnly := noderec.DirectoryNode{ - Name: "host-c", - IP: "10.0.0.6", - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: 14318}}, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: "uuid-c", + Name: "host-c", + IP: "10.0.0.6", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceNodeInfo: {Port: 14318}, + }, } _, ok = subscribedToNode(tc.profile, niOnly) require.False(t, ok, "node without this engine's service should not project") // Nor is a node running only the *other* engine. otherOnly := noderec.DirectoryNode{ - Name: "host-e", - IP: "10.0.0.8", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: "uuid-e", + Name: "host-e", + IP: "10.0.0.8", Services: map[noderec.ServiceKey]noderec.ServiceStatus{ other.DiscoveryService: {Port: other.FacadePort}, }, } _, ok = subscribedToNode(tc.profile, otherOnly) - require.False(t, ok, "a node running only") + require.False(t, ok, "node running only another engine must not project") // Per-engine attribution: a dual-engine node projects ONLY this // engine's models, never the cross-engine union, so a model served // solely via the other engine is not ranked as an owner here. dual := noderec.DirectoryNode{ - HostUUID: "uuid-d", - Name: "host-d", - IP: "10.0.0.7", - Models: []string{"mine", "theirs"}, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: "uuid-d", + Name: "host-d", + IP: "10.0.0.7", + Models: []string{ownModel, otherModel}, ModelsByEngine: map[string][]string{ - tc.profile.Name: {"mine"}, - other.Name: {"theirs"}, + tc.profile.Name: {ownModel}, + other.Name: {otherModel}, }, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ tc.profile.DiscoveryService: {Port: tc.profile.FacadePort}, @@ -157,7 +174,7 @@ func TestSubscribedToNode_ProjectsDiscoveryFields(t *testing.T) { } got, ok = subscribedToNode(tc.profile, dual) require.True(t, ok, "dual-engine node advertising this engine should project") - require.Equal(t, []string{"mine"}, got.Models, "dual-engine projection Models") + require.Equal(t, []string{ownModel}, got.Models, "dual-engine projection Models") }) } @@ -168,10 +185,13 @@ func TestSubscribedToNode_ProjectsDiscoveryFields(t *testing.T) { func TestSubscribedToNode_KeysByHostUUID(t *testing.T) { forEachEngine(t, func(t *testing.T, tc engineCase) { const uuid = "11111111-1111-1111-1111-111111111111" + const name = "host-a" n := noderec.DirectoryNode{ - HostUUID: uuid, - Name: "host-a", - IP: "10.0.0.5", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: uuid, + Name: name, + IP: "10.0.0.5", Services: map[noderec.ServiceKey]noderec.ServiceStatus{ tc.profile.DiscoveryService: {Port: tc.profile.FacadePort}, }, @@ -179,6 +199,6 @@ func TestSubscribedToNode_KeysByHostUUID(t *testing.T) { got, ok := subscribedToNode(tc.profile, n) require.True(t, ok, "node advertising this engine + IP should project") require.Equal(t, uuid, got.ID, "ID (%v)", uuid) - require.Equal(t, "host-a", got.Host) + require.Equal(t, name, got.Host) }) } diff --git a/services/nvpair-tui/ui/nodes.go b/services/nvpair-tui/ui/nodes.go index 41f68b3b8..12e755d6d 100644 --- a/services/nvpair-tui/ui/nodes.go +++ b/services/nvpair-tui/ui/nodes.go @@ -88,9 +88,8 @@ type nodesView struct { // invite sent by address, which has no node identity to key on — see // invitedAddress. invitedKey string - // invitedAddress is the host an invite-by-address is pending against. That - // path has no UUID, so the joined peer is recognised by its address instead; - // without it the PIN it pinned was never retired. + // invitedAddress is the target address, retained so a discovered row can be + // recognised after pairing replaces its unverified key with a host UUID. invitedAddress string // outboundInviteID is the invite our pending PIN belongs to. Terminal // notifications carry an inviteId and the manager supports concurrent @@ -223,8 +222,7 @@ type nodeInviteMsg struct { // inviteID identifies the session, so a later terminal notification can be // matched to the PIN this result pinned. inviteID string - // address is set when the invite went out by address rather than to a - // discovered node, which is how the joined peer is recognised later. + // address recognises the joined peer even if verification changes its row key. address string pin string rejected bool @@ -547,8 +545,8 @@ func (v *nodesView) handleInviteResult(msg nodeInviteMsg) tea.Cmd { // other machine. It clears when the invite resolves. // // The id is recorded so a terminal notification can be matched to this - // session, and the address so an invite sent by address — which has no - // node identity — can still recognise the peer once it joins. + // session, and the target address recognises the peer once it joins, + // even if pairing changes the discovery row's key. v.outboundInviteID = msg.inviteID v.invitedAddress = msg.address v.outboundName, v.outboundPIN = msg.name, msg.pin @@ -1013,12 +1011,12 @@ func (v *nodesView) inviteSelected() tea.Cmd { } params := map[string]any{"address": row.address, "nodeId": row.key} - name := row.name + name, address := row.name, row.address v.invitedKey = row.key v.inviteSending, v.outboundName = true, name v.status.busy("inviting %s...", name) return inviteNodeCmd(v.client, params, func(res inviteNodeResult, err error) tea.Msg { - return inviteResultMsg(name, "", res, err) + return inviteResultMsg(name, address, res, err) }) } @@ -1086,7 +1084,7 @@ func inviteNodeCmd(client *rpc.Client, params map[string]any, finish func(res in } // inviteResultMsg maps a cluster:invite-node outcome onto the view message. -// address is empty for an invite aimed at a discovered node. +// The target address remains usable when pairing changes a discovered row's key. func inviteResultMsg(name, address string, res inviteNodeResult, err error) tea.Msg { if err != nil { return nodeInviteMsg{name: name, address: address, err: err} @@ -1486,10 +1484,9 @@ func (v *nodesView) retirePendingInvite() { if n.membership != membershipMember { continue } - // Either identity works: a discovered node was invited by UUID, while an - // invite by address has none, so that peer is recognised by the address - // it was invited at. Without the address arm, a PIN pinned by the - // by-address path was never retired at all. + // A discovered row acquires its verified host UUID during pairing. The + // recorded target address recognises it after that key change, as well + // as a peer invited directly by address. This does not merge node rows. if (v.invitedKey != "" && n.key == v.invitedKey) || (v.invitedAddress != "" && addressMatches(n, v.invitedAddress)) { v.clearOutboundInvite() diff --git a/services/nvpair-tui/ui/nodesmodel.go b/services/nvpair-tui/ui/nodesmodel.go index 8419e18b7..54f5b41f2 100644 --- a/services/nvpair-tui/ui/nodesmodel.go +++ b/services/nvpair-tui/ui/nodesmodel.go @@ -7,6 +7,8 @@ import ( "net" "sort" "strings" + + "nvpair-shared/noderec" ) // Being listed and being reachable are different facts: a cluster member that @@ -100,9 +102,10 @@ func (m nodeMembership) relationship() string { // One machine can appear in all three. Merging on a stable key is what stops it // rendering as three separate rows. type nodeRow struct { - key string - name string - address string + identityVerification noderec.IdentityVerification + key string + name string + address string // addresses is every address the node published, ranked by the node itself // with address first. Anything that dials the node walks this rather than // assuming the first one is reachable from here. @@ -116,11 +119,9 @@ type nodeRow struct { // manualID is the handle node/remove needs. Empty unless the entry was // added by hand. manualID string - // nodeInfoTLS is a hand-added node that serves node-info over TLS. That - // endpoint needs the backend's cluster trust, which this client does not - // hold, so it is never polled from here: probedTelemetry, the manual-node - // worker's own reading, is shown instead, and is nil while the worker has - // none. + // nodeInfoTLS marks a manual TLS configuration. The TUI uses the worker's + // verified reading for this endpoint because the TUI does not hold the + // cluster certificate needed for a TLS request. nodeInfoTLS bool probedTelemetry *nodeTelemetry @@ -163,10 +164,14 @@ func mergeNodes(in nodeFeeds) []nodeRow { for _, d := range in.discovered { key := d.HostUUID + if d.IdentityVerification == noderec.IdentityUnverified { + key = d.DiscoveryRecordID + } if key == "" { key = "name:" + d.Name } row := get(key) + row.identityVerification = d.IdentityVerification row.name = d.Name row.address = d.IPAddress row.addresses = candidateAddresses(d) @@ -176,7 +181,7 @@ func mergeNodes(in nodeFeeds) []nodeRow { row.loadedByEngine = d.LoadedByEngine row.presence = discoveredPresence(d) switch { - case d.Trusted: + case d.Trusted && d.IdentityVerification != noderec.IdentityUnverified: row.membership = membershipMember case d.Clustered: row.membership = membershipForeign @@ -207,7 +212,7 @@ func mergeNodes(in nodeFeeds) []nodeRow { } for _, m := range in.manual { - row := matchManual(byKey, order, m) + row := matchManual(byKey, m) if row == nil { row = get("manual:" + m.ID) row.name = m.Name @@ -224,7 +229,7 @@ func mergeNodes(in nodeFeeds) []nodeRow { } if m.TLSEnabled { row.nodeInfoTLS = true - if m.NodeInfoUp { + if m.IdentityVerification == noderec.IdentityPinned && m.NodeInfoUp { row.probedTelemetry = &nodeTelemetry{ GPUs: m.GPUs, CPU: m.CPU, Memory: m.Memory, TelemetryValid: m.TelemetryValid, @@ -310,46 +315,13 @@ func candidateAddresses(d availableNode) []string { return out } -// matchManual finds the existing row a manual entry describes. -// -// The host UUID comes first, once the worker's probe has read one from the -// node's node-info. It is the only join that works when the operator typed a -// hostname and discovery reports the same machine by IP. -// -// Otherwise it falls back to the address, normalised — the manual one was typed -// by an operator, the discovered one comes off the wire — and compared against -// every address a node published, so a multi-homed node matches on any of -// them. -// -// An address is not an identity. Once the probe has read a UUID, the entry -// never joins a row that has a different UUID of its own: that is another -// machine now answering at the same address, and merging them would show one -// machine's membership and models under the other's name. -func matchManual(byKey map[string]*nodeRow, order []string, m manualNode) *nodeRow { - if m.HostUUID != "" { - if row, ok := byKey[m.HostUUID]; ok { - return row - } - } - want := normalizeHost(m.Address) - if want == "" { - return nil - } - for _, key := range order { - row := byKey[key] - if m.HostUUID != "" && hasHostIdentity(key) { - continue - } - if normalizeHost(row.address) == want { - return row - } - for _, candidate := range row.addresses { - if normalizeHost(candidate) == want { - return row - } - } +// matchManual joins only authenticated host identities. Configuration keys +// otherwise remain independent of names, addresses, and unverified UUID claims. +func matchManual(byKey map[string]*nodeRow, m manualNode) *nodeRow { + if m.IdentityVerification == noderec.IdentityPinned && m.HostUUID != "" { + return byKey[m.HostUUID] } - return nil + return byKey["manual:"+m.ID] } // hasHostIdentity reports whether a row key is a machine's own UUID, rather diff --git a/services/nvpair-tui/ui/nodesmodel_test.go b/services/nvpair-tui/ui/nodesmodel_test.go index ed29cd330..403c4857e 100644 --- a/services/nvpair-tui/ui/nodesmodel_test.go +++ b/services/nvpair-tui/ui/nodesmodel_test.go @@ -13,14 +13,45 @@ import ( "nvpair-shared/noderec" ) -// mergeReference is a fixed instant for building lastSeen timestamps. -// -// The merge itself no longer reads a clock — presence comes from whether the -// backend still lists a node, not from how old its record is — so this exists -// only to construct records of a given age and prove they are ignored. -var mergeReference = time.Unix(1_700_000_000, 0) +var mergeReference = time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant + +// A discovered candidate copying a member's UUID must keep its own row and +// remain outside membership. Its claimed identity cannot change the verified +// peer's endpoint or turn the candidate into that member. +func TestMergeNodes_UnverifiedClaimsRemainSeparateFromMembers(t *testing.T) { + const peerUUID = "peer" + const peerName = "verified" + const peerAddress = "192.0.2.20" + const candidateName = "rogue" + const candidateRecordID = "candidate:rogue" + rows := mergeNodes(nodeFeeds{ + discovered: []availableNode{ + { + HostUUID: peerUUID, + DiscoveryRecordID: "peer:peer", + IdentityVerification: noderec.IdentityPinned, + Name: peerName, + IPAddress: peerAddress, + Trusted: true, + }, + { + HostUUID: peerUUID, + DiscoveryRecordID: candidateRecordID, + IdentityVerification: noderec.IdentityUnverified, + Name: candidateName, + IPAddress: "192.0.2.30", + }, + }, + }) + + require.Len(t, rows, 2) + assert.Equal(t, membershipNone, findRow(t, rows, candidateName).membership) + assert.Equal(t, candidateRecordID, findRow(t, rows, candidateName).key) + assert.Equal(t, peerAddress, findRow(t, rows, peerName).address) +} -// seenAgo builds a lastSeen timestamp d before the reference instant. +// seenAgo builds a lastSeen timestamp d before a fixed reference instant. +// Presence comes from the service's snapshot, so these ages must not evict rows. func seenAgo(d time.Duration) int64 { return mergeReference.Add(-d).Unix() } func findRow(t *testing.T, rows []nodeRow, name string) nodeRow { @@ -137,22 +168,36 @@ func TestMerge_NeedsSomewhereToReachANode(t *testing.T) { // TestMergeDeduplicatesAcrossFeeds checks one machine appearing in all three // feeds renders as a single row carrying every feed's contribution. func TestMerge_DeduplicatesAcrossFeeds(t *testing.T) { + const hostUUID = "u1" + const hostName = "host" + const address = "10.0.0.5" + const manualID = "m1" rows := mergeNodes(nodeFeeds{ discovered: []availableNode{{ - HostUUID: "u1", - Name: "host", - IPAddress: "10.0.0.5", + HostUUID: hostUUID, + Name: hostName, + IPAddress: address, LastSeen: seenAgo(time.Second), Trusted: true, Models: []string{"llama3.2", "qwen3"}, }}, - members: []clusterNode{{NodeUUID: "u1", Name: "host", State: "joined"}}, - manual: []manualNode{{ID: "m1", Address: "10.0.0.5", NodeInfoUp: true}}, + members: []clusterNode{{ + NodeUUID: hostUUID, + Name: hostName, + State: "joined", + }}, + manual: []manualNode{{ + IdentityVerification: noderec.IdentityPinned, + HostUUID: hostUUID, + ID: manualID, + Address: address, + NodeInfoUp: true, + }}, }) require.Len(t, rows, 1, "one machine must produce one row") row := rows[0] - assert.Equal(t, "m1", row.manualID, "manual handle must survive the merge") + assert.Equal(t, manualID, row.manualID, "manual handle must survive the merge") assert.Equal(t, nodeMembership(membershipMember), row.membership) assert.Equal(t, 2, row.modelCount()) } @@ -180,29 +225,56 @@ func TestMergeManualProbe_BeatsDiscoverySilence(t *testing.T) { // joined to the same machine discovered by IP, and the node-info port and TLS // setting the probe found were dropped. func TestMerge_UsesEveryFactTheManualWorkerReports(t *testing.T) { + const manualID = "m1" + const hostUUID = "u1" + const lmName = "lms-only" + const tlsName = "tls-box" + const tlsPort = 14319 rows := mergeNodes(nodeFeeds{ - manual: []manualNode{{ID: "m1", Name: "lms-only", Address: "10.0.0.7", LMStudioUp: true}}, + manual: []manualNode{{ + ID: manualID, + Name: lmName, + Address: "10.0.0.7", + LMStudioUp: true, + }}, }) - assert.Equal(t, nodePresence(presenceOnline), findRow(t, rows, "lms-only").presence, "a host answering only on LM Studio must read online") + assert.Equal(t, nodePresence(presenceOnline), findRow(t, rows, lmName).presence, "a host answering only on LM Studio must read online") // Typed as a hostname, discovered by IP: only the UUID ties them together. rows = mergeNodes(nodeFeeds{ - discovered: []availableNode{{HostUUID: "u1", Name: "gpu-box", IPAddress: "10.0.0.5", Port: 14318}}, - manual: []manualNode{{ID: "m1", Address: "gpu-box.lan", HostUUID: "u1", NodeInfoUp: true}}, + discovered: []availableNode{{ + HostUUID: hostUUID, + Name: "gpu-box", + IPAddress: "10.0.0.5", + Port: 14318, + }}, + manual: []manualNode{{ + IdentityVerification: noderec.IdentityPinned, + ID: manualID, + Address: "gpu-box.lan", + HostUUID: hostUUID, + NodeInfoUp: true, + }}, }) require.Len(t, rows, 1, "the manual entry must be joined by its UUID") - assert.Equal(t, "m1", rows[0].manualID, "the joined row must retain its manual handle") + assert.Equal(t, manualID, rows[0].manualID, "the joined row must retain its manual handle") // A hand-added node reached on a non-default node-info port, over TLS. rows = mergeNodes(nodeFeeds{ manual: []manualNode{{ - ID: "m2", Name: "tls-box", Address: "10.0.0.9", NodeInfoUp: true, - NodeInfoPort: 14319, TLSEnabled: true, TelemetryValid: true, - GPUs: []noderec.GPUInfo{{Name: "GPU 0", VramBytes: 1 << 30}}, + IdentityVerification: noderec.IdentityPinned, + ID: "m2", + Name: tlsName, + Address: "10.0.0.9", + NodeInfoUp: true, + NodeInfoPort: tlsPort, + TLSEnabled: true, + TelemetryValid: true, + GPUs: []noderec.GPUInfo{{Name: "GPU 0", VramBytes: 1 << 30}}, }}, }) - tls := findRow(t, rows, "tls-box") - assert.Equal(t, 14319, tls.port, "node-info port must match the probe") + tls := findRow(t, rows, tlsName) + assert.Equal(t, tlsPort, tls.port, "node-info port must match the probe") assert.True(t, tls.nodeInfoTLS, "TLS node-info must be carried with the worker's reading") if assert.NotNil(t, tls.probedTelemetry, "worker's reading must be carried") { assert.Len(t, tls.probedTelemetry.GPUs, 1) @@ -212,24 +284,41 @@ func TestMerge_UsesEveryFactTheManualWorkerReports(t *testing.T) { // TestAReusedAddressDoesNotMergeTwoMachines checks an address is not taken for // an identity. A manual entry whose probe read one UUID must not fold into a // discovered row with a different UUID at the same address — that is another -// machine now answering there. An entry the probe has not identified still -// joins by address, as it must for a host discovery reports by IP. +// machine now answering there. An unidentified configuration also remains +// separate: sharing an address does not establish host identity. func TestAReusedAddress_DoesNotMergeTwoMachines(t *testing.T) { - discovered := []availableNode{{HostUUID: "machine-b", Name: "b", IPAddress: "10.0.0.5", Port: 14318}} + const address = "10.0.0.5" + const peerName = "b" + const manualID = "m1" + discovered := []availableNode{{ + HostUUID: "machine-b", + Name: peerName, + IPAddress: address, + Port: 14318, + }} rows := mergeNodes(nodeFeeds{ discovered: discovered, - manual: []manualNode{{ID: "m1", Name: "a", Address: "10.0.0.5", HostUUID: "machine-a", NodeInfoUp: true}}, + manual: []manualNode{{ + ID: manualID, + Name: "a", + Address: address, + HostUUID: "machine-a", + NodeInfoUp: true, + }}, }) require.Len(t, rows, 2, "two machines at one address must remain distinct") - assert.Empty(t, findRow(t, rows, "b").manualID, "machine b must not take the manual entry for machine a") + assert.Empty(t, findRow(t, rows, peerName).manualID, "machine b must not take the manual entry for machine a") rows = mergeNodes(nodeFeeds{ discovered: discovered, - manual: []manualNode{{ID: "m1", Address: "10.0.0.5"}}, + manual: []manualNode{{ + ID: manualID, + Address: address, + }}, }) - require.Len(t, rows, 1, "an unidentified entry must join the machine at its address") - assert.Equal(t, "m1", rows[0].manualID) + require.Len(t, rows, 2, "an address alone cannot join a configured target to another record") + assert.Empty(t, findRow(t, rows, peerName).manualID) } // TestTLSNodeInfoIsNotPolledInPlainText checks the detail screen shows the @@ -247,6 +336,35 @@ func TestTLSNodeInfo_IsNotPolledInPlainText(t *testing.T) { assert.Contains(t, d.View(), "GPU 0", "the worker's hardware reading must be shown") } +// Withdrawing a manual node's identity proof must remove its old hardware from +// an open detail screen without starting an HTTP poll against its TLS endpoint. +func TestOpenDetail_HidesManualHardwareAfterProofWithdrawal(t *testing.T) { + v := newNodesView(nil) + v.SetSize(100, 30) + manual := manualNode{ + ID: "tls-node", Name: "tls-box", Address: "10.0.0.9", + HostUUID: "peer", IdentityVerification: noderec.IdentityPinned, + OllamaUp: true, NodeInfoUp: true, NodeInfoPort: 14319, TLSEnabled: true, + TelemetryValid: true, + GPUs: []noderec.GPUInfo{{Name: "GPU 0", VramBytes: 1 << 30}}, + } + v.Update(manualNodesMsg{nodes: []manualNode{manual}}) + v.openDetail() + require.NotNil(t, v.detail) + require.Contains(t, v.View(), "GPU 0", "the authenticated hardware must be visible before proof withdrawal") + + manual.IdentityVerification = noderec.IdentityUnverified + manual.HostUUID = "" + manual.NodeInfoUp = false + manual.TelemetryValid = false + manual.GPUs = nil + v.Update(manualNodesMsg{nodes: []manualNode{manual}}) + + assert.Equal(t, presenceOnline, v.detail.node.presence, "the engine still answers after node-info proof is withdrawn") + assert.NotContains(t, v.View(), "GPU 0", "the old authenticated hardware must leave the open screen") + assert.Nil(t, v.detail.telemetryCmd(), "proof withdrawal must not start a plaintext poll against the configured TLS endpoint") +} + // TestMembershipGovernsInvitability is the guard for re-inviting a node that // already has a relationship. Only a standalone node may be invited. func TestNodeInvitation_RequiresEligibleClusterMembership(t *testing.T) { diff --git a/services/nvpair-tui/ui/nodeswire.go b/services/nvpair-tui/ui/nodeswire.go index a79bb72bc..ecd4ab853 100644 --- a/services/nvpair-tui/ui/nodeswire.go +++ b/services/nvpair-tui/ui/nodeswire.go @@ -84,9 +84,10 @@ type clusterInvite struct { // (services/nvpair-manual-nodes/manager.go), the element of the broker's // nodes/list reply, that the Nodes tab uses. type manualNode struct { - ID string `json:"id"` - Name string `json:"name"` - Address string `json:"address"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + ID string `json:"id"` + Name string `json:"name"` + Address string `json:"address"` // HostUUID is the machine's own identity, read from its node-info. It is // what joins a hand-typed hostname to the same machine discovered by IP. HostUUID string `json:"hostUuid"` diff --git a/services/nvpair-tui/ui/nodetelemetry.go b/services/nvpair-tui/ui/nodetelemetry.go index efd99d1fe..5af3e2fd1 100644 --- a/services/nvpair-tui/ui/nodetelemetry.go +++ b/services/nvpair-tui/ui/nodetelemetry.go @@ -115,10 +115,10 @@ func pollTelemetryCmd(key string, gen int, addresses []string, port int, expect var lastErr error for _, address := range addresses { t, err := fetchTelemetry(nodeInfoURL(address, port)) - if err == nil && expect != "" && t.HostUUID != "" && t.HostUUID != expect { + if err == nil && expect != "" && t.HostUUID != expect { // An address outlives the machine it was published for. Taken, // this showed another machine's hardware under this one's name. - err = fmt.Errorf("%s answered as another machine", address) + err = fmt.Errorf("%s did not identify the expected machine", address) } if err == nil { return nodeTelemetryMsg{nodeKey: key, gen: gen, address: address, telemetry: t} @@ -145,10 +145,10 @@ func preferAddress(addresses []string, last string) []string { } // telemetryIdentity is the machine a node's readings must come from, or empty -// when there is nothing to check an answer against: a row keyed by a stand-in -// rather than a UUID, or this machine, which is reached over loopback. +// when there is nothing to check an answer against: an unverified discovery or +// a row keyed by a stand-in rather than a UUID. Local rows also require a match. func telemetryIdentity(node nodeRow) string { - if node.self || !hasHostIdentity(node.key) { + if node.identityVerification == noderec.IdentityUnverified || !hasHostIdentity(node.key) { return "" } return node.key diff --git a/services/nvpair-tui/ui/nodetelemetry_test.go b/services/nvpair-tui/ui/nodetelemetry_test.go index a9275aa0f..3e54c2e9b 100644 --- a/services/nvpair-tui/ui/nodetelemetry_test.go +++ b/services/nvpair-tui/ui/nodetelemetry_test.go @@ -204,7 +204,8 @@ func TestTelemetryFromAnotherMachine_IsRefused(t *testing.T) { // A row with nothing to check against takes whichever machine answers. require.NoError(t, pollTelemetryCmd("manual:1", 1, []string{host}, port, "")().(nodeTelemetryMsg).err, "an unchecked poll was refused") - assert.Equal(t, "uuid-a", telemetryIdentity(nodeRow{key: "uuid-a"}), "a discovered node is checked against its UUID") + assert.Equal(t, "uuid-a", telemetryIdentity(nodeRow{key: "uuid-a", identityVerification: noderec.IdentityPinned}), "a verified node is checked against its UUID") + assert.Equal(t, "uuid-self", telemetryIdentity(nodeRow{key: "uuid-self", self: true, identityVerification: noderec.IdentityLocal})) test := func(name string, row nodeRow) { t.Run(name, func(t *testing.T) { assert.Empty(t, telemetryIdentity(row), "row has nothing to check") @@ -212,7 +213,29 @@ func TestTelemetryFromAnotherMachine_IsRefused(t *testing.T) { } test("manual entry", nodeRow{key: "manual:1"}) test("name-only entry", nodeRow{key: "name:lab"}) - test("self entry", nodeRow{key: "uuid-self", self: true}) +} + +func TestTelemetry_RequiresIdentityForVerifiedRows(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + assert.NoError(t, json.NewEncoder(w).Encode(map[string][]noderec.GPUInfo{"GPUs": {{Name: "Preview GPU"}}})) + })) + defer srv.Close() + host, port := splitTestServer(t, srv.URL) + test := func(name string, row nodeRow, wantError bool) { + t.Run(name, func(t *testing.T) { + msg := pollTelemetryCmd(row.key, 1, []string{host}, port, telemetryIdentity(row))().(nodeTelemetryMsg) + if wantError { + assert.Error(t, msg.err) + return + } + require.NoError(t, msg.err) + require.Len(t, msg.telemetry.GPUs, 1) + assert.Equal(t, "Preview GPU", msg.telemetry.GPUs[0].Name) + }) + } + test("unverified preview permits missing identity", nodeRow{key: "candidate:preview", identityVerification: noderec.IdentityUnverified}, false) + test("pinned row rejects missing identity", nodeRow{key: "peer-uuid", identityVerification: noderec.IdentityPinned}, true) + test("local row rejects missing identity", nodeRow{key: "self-uuid", self: true, identityVerification: noderec.IdentityLocal}, true) } // TestTelemetryTriesTheAddressThatAnswered is the regression guard for a poll diff --git a/services/nvpair-tui/ui/toast_test.go b/services/nvpair-tui/ui/toast_test.go index 6f466ba58..75c5851cb 100644 --- a/services/nvpair-tui/ui/toast_test.go +++ b/services/nvpair-tui/ui/toast_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "nvpair-shared/noderec" "nvpair-tui/rpc" tea "github.com/charmbracelet/bubbletea" @@ -129,6 +130,74 @@ func TestNodesView_RetiresPinOnPairingSuccess(t *testing.T) { assert.NotContains(t, v.status.render(), "123456", "PIN still on screen after pairing completed") } +// Pairing proves a host UUID, replacing the unverified discovery row's key. +// That change must retire the displayed PIN and allow another pairing. +func TestDiscoveredNodePairing_ClearsInviteAfterPeerJoins(t *testing.T) { + const discoveryRecordID = "observation:peer" + const peerUUID = "peer-uuid" + const peerAddress = "192.0.2.20" + const inviteID = "invite-peer" + clientConn, brokerConn := net.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancel() + assert.NoError(t, clientConn.Close()) + assert.NoError(t, brokerConn.Close()) + }) + require.NoError(t, brokerConn.SetDeadline(time.Now().Add(2*time.Second))) + client := rpc.NewClient(clientConn, clientConn) + go client.Run(ctx) + broker := rpc.NewCodec(brokerConn, brokerConn) + v := newNodesView(client) + v.feeds.discovered = []availableNode{{ + DiscoveryRecordID: discoveryRecordID, IdentityVerification: noderec.IdentityUnverified, + HostUUID: peerUUID, Name: "peer", IPAddress: peerAddress, Port: 14318, + }} + v.rebuild() + v.selectedKey = discoveryRecordID + pressPair := func() tea.Cmd { + return v.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("p")}) + } + cmd := pressPair() + require.NotNil(t, cmd) + results := make(chan tea.Msg, 1) + go func() { results <- cmd() }() + request, err := broker.Read() + require.NoError(t, err) + require.Equal(t, "cluster:invite-node", request.Method) + require.NoError(t, broker.Write(&rpc.Message{ + JSONRPC: "2.0", ID: request.ID, + Result: json.RawMessage(`{"inviteId":"invite-peer","state":"pending","pin":"123456"}`), + })) + select { + case result := <-results: + v.Update(result) + case <-time.After(2 * time.Second): + t.Fatal("pairing did not report its pending invitation") + } + require.Equal(t, inviteID, v.outboundInviteID) + require.Contains(t, v.status.render(), "123456") + + v.feeds.discovered = []availableNode{ + { + DiscoveryRecordID: "peer:verified", IdentityVerification: noderec.IdentityPinned, + HostUUID: peerUUID, Name: "peer", IPAddress: peerAddress, Port: 14318, Trusted: true, + }, + { + DiscoveryRecordID: "observation:next", IdentityVerification: noderec.IdentityUnverified, + HostUUID: "next-uuid", Name: "next", IPAddress: "192.0.2.30", Port: 14318, + }, + } + v.feeds.members = []clusterNode{{NodeUUID: peerUUID, Name: "peer", State: "member"}} + v.rebuild() + + assert.Empty(t, v.outboundInviteID) + assert.Empty(t, v.invitedKey) + assert.NotContains(t, v.status.render(), "123456") + v.selectedKey = "observation:next" + assert.NotNil(t, pressPair(), "a completed pairing must not block the next invitation") +} + // TestNodesViewKeepsPinWhileInvitePending checks an unrelated snapshot does not // retire a PIN that is still live. func TestNodesView_KeepsPinWhileInvitePending(t *testing.T) { diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 78183e1bf..5c1f736c2 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -51,11 +51,18 @@ Two responsibilities live in the broker itself rather than in a worker: feeds the scheduler, and forwards them to the workload manager for broadcast. Peer-origin updates come back the other way as `workloads:upsert` / `workloads:remove` for subscribed clients. -- **Scheduling telemetry.** Scanner and manual-node observations populate a +- **Scheduling telemetry.** Owned local and pinned remote observations populate a source-aware cache keyed by `hostUuid`; scanner data takes precedence. The broker advances sample age locally, feeds compact maximum-GPU telemetry to the scheduler, and replays it after a scheduler restart. +Unverified discoveries retain independent `discoveryRecordId` keys throughout +the store and relay. They cannot overwrite a verified host, clear its telemetry, +fail its workloads, or enter the scheduler's node universe. Explicit manual +engine targets keep their own operational keys until authenticated. The broker +caches and forwards scanner `discovery:node-info` hardware snapshots only when +their record identity matches the current verified scanner record. + ### Supervision & recovery Every worker runs under a supervisor that auto-restarts it on an unexpected exit: exponential backoff (~1 s → 16 s), a budget of 5 attempts per unhealthy streak, and a 60 s "healthy reset" window after which a restarted worker is considered recovered. A crash is surfaced as a sticky `supervisor:subprocess-crashed:` entry in the error registry (so connected tools can show "this helper is down"); recovery clears it, and a worker that exhausts its restart budget leaves the entry up. The scanner is the one worker whose **initial** spawn is fatal (the broker has no job without discovery); every other worker is optional and a failed first spawn just runs the broker without it. `nvpair-errors` is a special case: it can't report its own death into itself, so its crash logs to stderr (rather than the pipeline) while still being auto-restarted. @@ -106,7 +113,7 @@ Logs go to **stderr** (shared `applog` format, same as every other NVPAIR binary On startup — **before** emitting `app:ready` — the broker spawns the scanner and (when available) node-info, `nvpair-proxy`, the workload-manager, and the cluster-manager as child processes over stdio. The proxy is spawned up front but doesn't gate `app:ready` — each of its facades announces its listen port asynchronously (see below). None of the auxiliary workers gate `app:ready`. -**`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by `id`. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm`/`lc` from the engine poller) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. +**`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by verified host identity or independent discovery record identity. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm`/`lc` from the engine poller) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. **`nvpair-node-info`** serves public display HTTP and peer node inventory over pinned mTLS on its registered `ni` port. The broker passes its cluster directory and host UUID, @@ -562,7 +569,9 @@ Any `settings/*` request is forwarded to `nvpair-node-settings` and its response #### `node/add` / `node/remove` / `nodes/list` (manual nodes) -Relayed to `nvpair-manual-nodes`. `node/add` (`{ address, name?, tls_port?, mtls? }`) registers a user-added node and probes it; `node/remove` (`{ id }`) drops it; `nodes/list` returns the tracked manual nodes. Manually added nodes also surface in the shared `discovery:get-nodes` / `discovery:nodes-changed` snapshot — the broker merges `nvpair-manual-nodes`' `node/discovered|updated|removed` into the same store the scanner feeds. The broker persists successful `node/add` operations to `configs/manual-nodes.json` in the per-user data directory and replays them into a replacement `nvpair-manual-nodes` worker after startup. Successful `node/remove` operations update the same durable list. Error `-32000 "manual-nodes not available"` when no manual-nodes worker is supervised. +Relayed to `nvpair-manual-nodes`. `node/add` (`{ address, name?, tls_port?, mtls? }`) registers a user-added node and probes it; `node/remove` drops it; `nodes/list` returns the tracked manual nodes. For removal, supply exactly one of `{ id }` (the worker's exact entry ID, as used by the TUI) or `{ nodeId }` (the broker's node key, as used by the desktop). The broker resolves `nodeId` to an entry ID before forwarding `{ id }` to the worker. An unknown node key returns `{ removed: false }` without contacting the worker or changing saved entries. Missing or conflicting selectors return error `-32602`. + +Manually added nodes also surface in the shared `discovery:get-nodes` / `discovery:nodes-changed` snapshot — the broker merges `nvpair-manual-nodes`' `node/discovered|updated|removed` into the same store the scanner feeds. The broker persists successful `node/add` operations to `configs/manual-nodes.json` in the per-user data directory and replays them into a replacement `nvpair-manual-nodes` worker after startup. Successful `node/remove` operations update the same durable list. Error `-32000 "manual-nodes not available"` when no manual-nodes worker is supervised. **Manual → proxy bridge.** When the broker supervises both `nvpair-manual-nodes` and a proxy, it also bridges a manual node whose engine is reachable into that proxy via `node/add-manual` (host/port from the node's per-engine status), so inference can route to it through `ollama-proxy:node/select` / `lmstudio-proxy:node/select` just like a relay-discovered node. This is per-engine: a node whose `ollama_*` status is up is bridged into `ollama-proxy` (host/port from `ollama_port`), and one whose `lmstudio_*` status is up into `lmstudio-proxy` (from `lmstudio_port`) — a node running both is bridged into both. Manual nodes are by definition the ones that never appear via the daemon's `_nvpair-node` discovery, so this explicit add is what makes them routable. The bridge tracks reachability: an engine that goes down (or a node that is removed, or whose prober crashes) is pulled back out with `node/remove-manual`. A proxy that isn't supervised → that leg is a no-op; manual nodes still appear in the discovery snapshot as before. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 9e44ec13a..72b9e4cac 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -629,6 +629,15 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { func(telemetry noderec.NodeTelemetry) { b.ingestTelemetry(sourceScanner, telemetry) }, func(hostUUID string) { b.removeTelemetry(sourceScanner, hostUUID) }, b.updateLocalPeerMetadata, + func(value noderec.NodeInfoEvent) { + cached, ok := b.store.nodeInfo(value.HostUUID, time.Now()) + if !ok { + return + } + if err := b.codec.Notify(noderec.NotifyNodeInfo, cached); err != nil { + slog.Warn("forward node-info failed", "host_uuid", value.HostUUID, "err", err) + } + }, b.clusterDirArgs()..., ) if err != nil { @@ -1200,7 +1209,7 @@ func (b *Broker) spawnJobScheduler() (supervisedHandle, error) { b.setScheduler(w) replayed := b.replayActiveWorkloadsToScheduler(w) telemetryReplayed := b.replayTelemetryToScheduler(w) - if err := w.Notify("discovery:nodes-changed", b.store.Snapshot()); err != nil { + if err := w.Notify("discovery:nodes-changed", b.store.SchedulingSnapshot()); err != nil { slog.Warn("push discovery baseline to scheduler failed", "err", err) } b.schedulerFeedMu.Unlock() @@ -1604,7 +1613,11 @@ func (b *Broker) upsertManualNode(s manualNodeStatus) { b.manualMu.Unlock() b.store.Upsert(en, sourceManual) - b.ingestTelemetryAt(sourceManual, manualNodeTelemetry(s, key), receivedAt) + if s.IdentityVerification == noderec.IdentityPinned && s.HostUUID != "" { + b.ingestTelemetryAt(sourceManual, manualNodeTelemetry(s, s.HostUUID), receivedAt) + } else { + b.removeTelemetry(sourceManual, key) + } // Bridge a reachable manual node into each engine's proxy (ollama-proxy / // lmstudio-proxy) so inference can route to it; an unreachable engine is // pulled back out. No-op for a proxy the broker doesn't supervise. @@ -1650,7 +1663,9 @@ func (b *Broker) reprojectOrRelease(key string) { if ok { b.store.Upsert(manualToEnriched(survivor.status), sourceManual) b.bridgeManualNode(survivor.status, key) - b.ingestTelemetryAt(sourceManual, manualNodeTelemetry(survivor.status, key), survivor.receivedAt) + if survivor.status.IdentityVerification == noderec.IdentityPinned { + b.ingestTelemetryAt(sourceManual, manualNodeTelemetry(survivor.status, key), survivor.receivedAt) + } return } b.store.Remove(key, sourceManual) @@ -2330,9 +2345,11 @@ func (b *Broker) emitNodesChanged() { b.pushNodesChanged() } -// fanDiscoveryToScheduler pushes the current discovery snapshot to the scheduler -// child as a discovery:nodes-changed notification (same bare-array shape the -// client stream uses). A no-op when no scheduler is running. +// fanDiscoveryToScheduler sends SchedulingSnapshot to the scheduler child as a +// discovery:nodes-changed notification. That snapshot includes local and verified +// discoveries plus explicitly configured manual engine targets, excluding +// unverified scanner claims. It uses the client's bare-array wire shape and is +// a no-op when no scheduler is running. func (b *Broker) fanDiscoveryToScheduler() { b.schedulerFeedMu.Lock() defer b.schedulerFeedMu.Unlock() @@ -2341,7 +2358,7 @@ func (b *Broker) fanDiscoveryToScheduler() { if sc == nil { return } - if err := sc.Notify("discovery:nodes-changed", b.store.Snapshot()); err != nil { + if err := sc.Notify("discovery:nodes-changed", b.store.SchedulingSnapshot()); err != nil { slog.Warn("fan discovery:nodes-changed to scheduler failed", "err", err) } } @@ -3047,7 +3064,7 @@ func (b *Broker) nodeDisplayNames() map[string]string { nodes := b.store.Snapshot() names := make(map[string]string, len(nodes)) for _, n := range nodes { - if n.HostUUID != "" && n.Name != "" { + if n.IdentityVerification != noderec.IdentityUnverified && n.HostUUID != "" && n.Name != "" { names[n.HostUUID] = n.Name } } @@ -3454,6 +3471,15 @@ func (b *Broker) relayToManualNodes(msg *Message) { id := msg.ID method := msg.Method params := append(json.RawMessage(nil), msg.Params...) + var remove manualNodeRemoveParams + if method == "node/remove" { + if err := json.Unmarshal(params, &remove); err != nil || (remove.ID == "") == (remove.NodeID == "") { + if err := b.codec.RespondError(id, -32602, "exactly one of id or nodeId is required"); err != nil { + log.Printf("failed to respond to invalid node/remove: %v", err) + } + return + } + } mutates := method == "node/add" || method == "node/remove" if mutates { b.manualMutationMu.Lock() @@ -3464,13 +3490,18 @@ func (b *Broker) relayToManualNodes(msg *Message) { _ = json.Unmarshal(params, &added) } if method == "node/remove" { - var remove struct { - ID string `json:"id"` + removeID = remove.ID + if remove.NodeID != "" { + removeID = b.manualAliasForKey(remove.NodeID) } - if json.Unmarshal(params, &remove) == nil && remove.ID != "" { - removeID = b.manualAliasForKey(remove.ID) - params, _ = json.Marshal(map[string]string{"id": removeID}) + if removeID == "" { + b.manualMutationMu.Unlock() + if err := b.codec.Respond(id, map[string]bool{"removed": false}); err != nil { + log.Printf("failed to respond to unmatched node/remove: %v", err) + } + return } + params, _ = json.Marshal(manualNodeRemoveParams{ID: removeID}) } relayErr := mn.RelayRequest(method, params, func(result json.RawMessage, rpcErr *RPCError, err error) { if mutates { diff --git a/services/nvpair-ui-broker/broker_test.go b/services/nvpair-ui-broker/broker_test.go new file mode 100644 index 000000000..c2346d6e1 --- /dev/null +++ b/services/nvpair-ui-broker/broker_test.go @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "nvpair-shared/noderec" +) + +// Give the broker a verified peer, an unverified record claiming that peer's +// UUID, and an explicitly configured manual target. Read the scheduler's actual +// notification and check that it contains only the peer and manual target. +// A manual configuration authorizes routing without authorizing hardware data. +func TestFanDiscoveryToScheduler_ExcludesUnverifiedDiscoveries(t *testing.T) { + const peerUUID = "peer" + const verifiedName = "verified" + const manualID = "configured" + brokerSide, schedulerSide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, brokerSide.Close()) + assert.NoError(t, schedulerSide.Close()) + }) + broker := newManualTestBroker() + broker.setScheduler(&rpcWorker{peer: NewPeer(NewCodec(brokerSide))}) + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, + ID: verifiedName, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + }, sourceScanner) + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, + ID: "rogue", + DiscoveryRecordID: "candidate:rogue", + IdentityVerification: noderec.IdentityUnverified, + }, sourceScanner) + broker.store.Upsert(manualToEnriched(manualNodeStatus{ + ID: manualID, + Address: "192.0.2.30", + }), sourceManual) + // net.Pipe writes wait for a reader, so fanout must run alongside the read + // below. The completion channel lets us join the writer before pipe cleanup. + done := make(chan struct{}) + go func() { + broker.fanDiscoveryToScheduler() + close(done) + }() + require.NoError(t, schedulerSide.SetReadDeadline(time.Now().Add(2*time.Second))) + + message, err := NewCodec(schedulerSide).Read() + require.NoError(t, err) + var nodes []AvailableNode + require.NoError(t, json.Unmarshal(message.Params, &nodes)) + require.Len(t, nodes, 2) + assert.Equal(t, manualID, nodes[0].ID) + assert.Equal(t, "manual:configured", nodes[0].HostUUID) + assert.Equal(t, verifiedName, nodes[1].ID) + <-done +} diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 510729f18..054652951 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -37,12 +37,11 @@ type EnrichedNode struct { DiscoveryRecordID string `json:"discoveryRecordId"` IdentityVerification noderec.IdentityVerification `json:"identityVerification"` ID string `json:"id"` - // HostUUID is the node's stable per-host identity (the daemon's uuid= TXT). - // It's the discovery-store key, so a PC rename — which changes the hostname - // (ID) but not the UUID — updates the existing entry in place instead of - // leaving a ghost under the old name. It stays off the wire; the - // client-facing id/name remain the hostname. Empty for manual nodes, which - // fall back to keying by their own ID. + // HostUUID identifies the host for local or pinned records. + // For unverified mDNS records it is the advertised UUID, which is only a claim. + // For unverified manual entries it is "manual:" + the entry ID, matching the + // configured engine route's key. The store uses DiscoveryRecordID for both + // kinds of unverified record so they stay separate from verified hosts. HostUUID string `json:"-"` Host string `json:"host"` Port int `json:"port"` @@ -75,11 +74,8 @@ type EnrichedNode struct { LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` } -// directoryToEnriched projects the promoted daemon's DirectoryNode onto the -// broker's EnrichedNode so the client-facing store (which already merges manual -// nodes and serves discovery:get-nodes) can be fed by the new pipeline. ID is -// the mDNS instance name (hostname), matching the legacy node/* key so the two -// feeds dedup during the migration; Port is the node-info port. +// directoryToEnriched preserves scanner provenance in the client-facing store. +// ID remains the display name; Port is the node-info port. // // The node's whole ranked address list travels through, both as Addresses and as // the ip= / ips= TXT the netpick projection reads, so its own order survives to @@ -116,14 +112,13 @@ func directoryToEnriched(n noderec.DirectoryNode) EnrichedNode { } } -// storeKey is the discovery-store key for a node: its stable host UUID. Every -// node carries one by the time it reaches the store — a real UUID for mDNS and -// manual nodes alike (manual nodes learn theirs from the peer's node-info), -// computed at each ingestion boundary — so keying is uniform and a hostname -// change never duplicates or splits an entry. No id fallback: an empty -// key is rejected by Upsert rather than silently keyed by name. +// storeKey merges only proved host identities. Unverified discoveries and +// manual configurations retain independent record keys. func (n EnrichedNode) storeKey() string { - return n.HostUUID + if n.IdentityVerification == noderec.IdentityLocal || n.IdentityVerification == noderec.IdentityPinned { + return n.HostUUID + } + return n.DiscoveryRecordID } // nodeSource identifies which feed contributed a store record. The same node @@ -147,6 +142,74 @@ type storedNode struct { scanner *EnrichedNode manual *EnrichedNode lastSeen time.Time + info *nodeInfoObservation +} + +type nodeInfoObservation struct { + value noderec.NodeInfoEvent + receivedAt time.Time +} + +// AcceptInfo binds the complete snapshot to the scanner's current record. A +// candidate with the same claimed UUID cannot populate this cache. +func (s *discoveryStore) AcceptInfo(value noderec.NodeInfoEvent, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + sn, ok := s.nodes[value.HostUUID] + if !ok || sn.scanner == nil { + return false + } + node := sn.scanner + if node.DiscoveryRecordID != value.DiscoveryRecordID || node.IdentityVerification != value.IdentityVerification || value.Info.HostUUID != value.HostUUID { + return false + } + if value.IdentityVerification != noderec.IdentityLocal { + if value.IdentityVerification != noderec.IdentityPinned || !node.Trusted { + // Remote hardware needs both proved identity and current trust. + return false + } + if value.ClusterUUID != value.HostUUID { + // The event's authenticated principal must identify this same host. + return false + } + if value.Info.ClusterUUID == nil || *value.Info.ClusterUUID != value.ClusterUUID { + // The hardware body must name that principal too; a missing or + // conflicting identity cannot populate the verified cache. + return false + } + } + sn.info = &nodeInfoObservation{value: value, receivedAt: now} + s.nodes[value.HostUUID] = sn + return true +} + +func (s *discoveryStore) nodeInfo(hostUUID string, now time.Time) (noderec.NodeInfoEvent, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + sn, ok := s.nodes[hostUUID] + if !ok || sn.info == nil { + return noderec.NodeInfoEvent{}, false + } + value := sn.info.value + value.Info = value.Info.Aged(now.Sub(sn.info.receivedAt)) + return value, true +} + +func (s *discoveryStore) verifiedSource(hostUUID string, source nodeSource) bool { + s.mu.RLock() + defer s.mu.RUnlock() + sn := s.nodes[hostUUID] + node := sn.scanner + if source == sourceManual { + node = sn.manual + } + if node == nil { + return false + } + if node.IdentityVerification == noderec.IdentityLocal { + return true + } + return node.IdentityVerification == noderec.IdentityPinned && node.Trusted } // projected returns the EnrichedNode a consumer sees. When both sources claim @@ -262,6 +325,9 @@ func (s *discoveryStore) Remove(key string, source nodeSource) bool { return false } sn.setSource(source, nil) + if source == sourceScanner { + sn.info = nil + } gone := !sn.claimed() if gone { delete(s.nodes, key) @@ -283,13 +349,42 @@ func (s *discoveryStore) Remove(key string, source nodeSource) bool { // for future RPCs; only the fields the external schema requires are // projected out here. func (s *discoveryStore) Snapshot() []AvailableNode { + return s.snapshot(false) +} + +// SchedulingSnapshot includes local discoveries, pinned and trusted peers, +// and explicitly configured manual engine targets. Unverified scanner claims +// are excluded. Manual hardware still requires separate authentication. +func (s *discoveryStore) SchedulingSnapshot() []AvailableNode { + return s.snapshot(true) +} + +func (s *discoveryStore) snapshot(scheduling bool) []AvailableNode { s.mu.RLock() out := make([]AvailableNode, 0, len(s.nodes)) for _, sn := range s.nodes { + // Explicit manual configurations independently authorize engine routing. + if scheduling && sn.manual == nil { + verification := sn.scanner.IdentityVerification + if verification != noderec.IdentityLocal && verification != noderec.IdentityPinned { + // An unverified advertisement cannot create a scheduling target. + continue + } + if verification == noderec.IdentityPinned && !sn.scanner.Trusted { + // Proving a remote identity is insufficient without current trust. + // Local records come from the owned worker and need no peer pin. + continue + } + } out = append(out, sn.toAvailable()) } s.mu.RUnlock() - sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + sort.Slice(out, func(i, j int) bool { + if out[i].ID == out[j].ID { + return out[i].DiscoveryRecordID < out[j].DiscoveryRecordID + } + return out[i].ID < out[j].ID + }) return out } @@ -324,7 +419,7 @@ func (sn storedNode) toAvailable() AvailableNode { IPAddresses: candidates, Port: n.Port, LastSeen: sn.lastSeen.Unix(), - Trusted: n.Trusted, + Trusted: n.Trusted && n.IdentityVerification != noderec.IdentityUnverified, Clustered: n.Clustered, Models: n.Models, ModelsByEngine: n.ModelsByEngine, @@ -355,6 +450,7 @@ type scannerProcess struct { onTelemetry func(noderec.NodeTelemetry) onTelemetryRemoved func(hostUUID string) onLocalNode func(noderec.DirectoryNode) + onNodeInfo func(noderec.NodeInfoEvent) localInfo chan localInfoObservation // activity queues peer liveness reports for the scanner. It exists so the // proxy-reader goroutine that receives them never waits on the scanner's @@ -403,6 +499,7 @@ func startScanner( onTelemetry func(noderec.NodeTelemetry), onTelemetryRemoved func(hostUUID string), onLocalNode func(noderec.DirectoryNode), + onNodeInfo func(noderec.NodeInfoEvent), extraArgs ...string, ) (*scannerProcess, error) { cmd := exec.Command(binaryPath, append([]string{"--log-level", logLevel}, extraArgs...)...) @@ -434,6 +531,7 @@ func startScanner( onTelemetry: onTelemetry, onTelemetryRemoved: onTelemetryRemoved, onLocalNode: onLocalNode, + onNodeInfo: onNodeInfo, localInfo: make(chan localInfoObservation, 1), activity: make(chan nodeActivityReport, activityQueueDepth), done: make(chan struct{}), @@ -609,7 +707,8 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { if s.relayDir != nil { s.relayDir.Apply(method, ev.Node) } - if method == noderec.NotifyNodeRemoved && s.onTelemetryRemoved != nil { + verified := ev.Node.IdentityVerification == noderec.IdentityLocal || ev.Node.IdentityVerification == noderec.IdentityPinned + if method == noderec.NotifyNodeRemoved && verified && s.onTelemetryRemoved != nil { s.onTelemetryRemoved(ev.Node.HostUUID) } if s.store != nil { @@ -623,7 +722,7 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { // present, so we must not fail its jobs. Identify the departed // node by HostUUID, the value workloads are stamped/keyed with, // not the display hostname. - if s.store.Remove(directoryToEnriched(ev.Node).storeKey(), sourceScanner) && s.onNodeLost != nil { + if s.store.Remove(directoryToEnriched(ev.Node).storeKey(), sourceScanner) && verified && s.onNodeLost != nil { s.onNodeLost(ev.Node.HostUUID, ev.Node.Name) } } else { @@ -640,9 +739,17 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { slog.Warn("daemon emitted node telemetry without hostUuid") return } - if s.onTelemetry != nil { + if s.store != nil && s.store.verifiedSource(telemetry.HostUUID, sourceScanner) && s.onTelemetry != nil { s.onTelemetry(telemetry) } + case noderec.NotifyNodeInfo: + var value noderec.NodeInfoEvent + if json.Unmarshal(params, &value) != nil || s.store == nil || !s.store.AcceptInfo(value, time.Now()) { + return + } + if s.onNodeInfo != nil { + s.onNodeInfo(value) + } case "ready": slog.Info("scanner reported ready", "params", string(params)) } diff --git a/services/nvpair-ui-broker/discovery_test.go b/services/nvpair-ui-broker/discovery_test.go index 8caec6200..70513bbfa 100644 --- a/services/nvpair-ui-broker/discovery_test.go +++ b/services/nvpair-ui-broker/discovery_test.go @@ -6,6 +6,7 @@ package main import ( "encoding/json" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -13,46 +14,191 @@ import ( "nvpair-shared/noderec" ) +// Store a verified peer alongside an unverified advertisement copying its UUID +// and name. Removing the advertisement must leave the peer's record, endpoint, +// and trust intact, rather than overwriting or removing that peer. +func TestDiscoveryStore_IsolatesUnverifiedClaimsFromVerifiedPeers(t *testing.T) { + store := newDiscoveryStore() + const knownUUID = "host-a" + const peerRecordID = "peer:host-a" + const peerName = "peer" + const peerAddress = "192.0.2.20" + peer := noderec.DirectoryNode{ + DiscoveryRecordID: peerRecordID, + HostUUID: knownUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + Name: peerName, + IP: peerAddress, + } + candidate := noderec.DirectoryNode{ + DiscoveryRecordID: "candidate:rogue", + HostUUID: knownUUID, + IdentityVerification: noderec.IdentityUnverified, + Name: peerName, + IP: "192.0.2.30", + } + store.Upsert(directoryToEnriched(peer), sourceScanner) + store.Upsert(directoryToEnriched(candidate), sourceScanner) + + require.Len(t, store.Snapshot(), 2) + store.Remove(candidate.DiscoveryRecordID, sourceScanner) + snapshot := store.Snapshot() + require.Len(t, snapshot, 1) + assert.Equal(t, peerRecordID, snapshot[0].DiscoveryRecordID) + assert.Equal(t, peerAddress, snapshot[0].IPAddress) + assert.True(t, snapshot[0].Trusted) +} + +// Process a removal for an unverified record that claims a host UUID. Neither +// host-level callback may fire: clearing telemetry or failing workloads would +// incorrectly attribute the candidate's disappearance to that claimed host. +func TestScannerProcess_CandidateRemovalPreservesPeerTelemetryAndWorkloads(t *testing.T) { + var removed, lost int + store := newDiscoveryStore() + sp := &scannerProcess{ + store: store, + onTelemetryRemoved: func(string) { removed++ }, + onNodeLost: func(string, string) { lost++ }, + } + candidate := noderec.DirectoryNode{ + DiscoveryRecordID: "candidate:rogue", + HostUUID: "host-a", + IdentityVerification: noderec.IdentityUnverified, + } + store.Upsert(directoryToEnriched(candidate), sourceScanner) + params, err := json.Marshal(noderec.NodeEvent{Node: candidate}) + require.NoError(t, err) + + sp.handleNotify(noderec.NotifyNodeRemoved, params) + + assert.Zero(t, removed) + assert.Zero(t, lost) +} + +// Accept hardware for the current verified record, then attempt replacement +// under a different record ID. The cache must preserve and age the accepted +// sample, invalidate it on removal, and reject a late sample after removal. +func TestDiscoveryStore_CachesSnapshotsOnlyForTheirVerifiedRecord(t *testing.T) { + const hostUUID = "host-a" + const peerRecordID = "peer:host-a" + store := newDiscoveryStore() + store.Upsert(EnrichedNode{ + HostUUID: hostUUID, + DiscoveryRecordID: peerRecordID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + }, sourceScanner) + principal := hostUUID + value := noderec.NodeInfoEvent{ + HostUUID: hostUUID, + ClusterUUID: principal, + DiscoveryRecordID: peerRecordID, + IdentityVerification: noderec.IdentityPinned, + Info: noderec.NodeInfo{ + HostUUID: hostUUID, + ClusterUUID: &principal, + TelemetryValid: true, + MSSince: 100, + GPUs: []GPUInfo{{Name: "accepted GPU"}}, + }, + } + now := time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant + // The snapshot names the same host, verified record, and pinned principal + // as the scanner's current directory entry, so it may populate the cache. + require.True(t, store.AcceptInfo(value, now)) + forged := value + forged.DiscoveryRecordID = "candidate:rogue" + forged.Info.GPUs = []GPUInfo{{Name: "forged GPU"}} + // A matching host UUID and a claimed "pinned" flag cannot authorize a + // different discovery record to replace that host's accepted measurements. + assert.False(t, store.AcceptInfo(forged, now.Add(time.Second))) + accepted, ok := store.nodeInfo(hostUUID, now.Add(2*time.Second)) + // Rejecting the forged update preserves the original verified snapshot. + require.True(t, ok) + assert.Equal(t, value.Info.GPUs, accepted.Info.GPUs) + // Its original 100 ms sample age includes the two seconds spent cached. + assert.Equal(t, int64(2100), accepted.Info.MSSince) + + // Removing the verified scanner record also invalidates its cached data. + store.Remove(hostUUID, sourceScanner) + _, cached := store.nodeInfo(hostUUID, now.Add(3*time.Second)) + assert.False(t, cached) + // A late snapshot cannot recreate the missing verified directory record. + assert.False(t, store.AcceptInfo(value, now.Add(3*time.Second))) +} + // TestDiscoveryStoreRekeysOnRename verifies that a discovered node changing // hostname (same host UUID, new mDNS instance name) updates its single // store entry in place rather than leaving a ghost under the old name. func TestDiscoveryStore_RekeysOnRename(t *testing.T) { s := newDiscoveryStore() const uuid = "11111111-1111-1111-1111-111111111111" - - s.Upsert(EnrichedNode{ID: "old-host", HostUUID: uuid}, sourceScanner) - s.Upsert(EnrichedNode{ID: "new-host", HostUUID: uuid}, sourceScanner) + const newName = "new-host" + + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: "old-host", + HostUUID: uuid, + }, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: newName, + HostUUID: uuid, + }, sourceScanner) snap := s.Snapshot() require.Len(t, snap, 1, "a rename must not duplicate the node") - require.Equal(t, "new-host", snap[0].ID, "store did not track the new name") + require.Equal(t, newName, snap[0].ID, "store did not track the new name") require.Equal(t, uuid, snap[0].HostUUID, "hostUuid not projected to the wire") } // TestDiscoveryStoreDistinctUUIDsSameName verifies the same-hostname collision // case: two different machines that share a hostname stay two entries. func TestDiscoveryStore_KeepsNodesWithSameNameAndDifferentUUIDs(t *testing.T) { + const name = "samename" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "samename", HostUUID: "aaaaaaaa-0000-0000-0000-000000000001"}, sourceScanner) - s.Upsert(EnrichedNode{ID: "samename", HostUUID: "bbbbbbbb-0000-0000-0000-000000000002"}, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: "aaaaaaaa-0000-0000-0000-000000000001", + }, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: "bbbbbbbb-0000-0000-0000-000000000002", + }, sourceScanner) require.Len(t, s.Snapshot(), 2, "two machines sharing a hostname must not merge") } -// TestManualToEnrichedHostUUID verifies the manual-node ingestion boundary: a -// manual node keys by the remote's real hostUuid once node-info reports one, and -// by its manual id until then. -func TestDiscoveryStore_RekeysManualNodeToLearnedHostUUID(t *testing.T) { - withUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9", HostUUID: "real-uuid"}) - require.Equal(t, "real-uuid", withUUID.storeKey(), "store key") - noUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9"}) - require.Equal(t, "manual:10.0.0.9", noUUID.storeKey(), "store key") +func TestManualToEnriched_UnverifiedHostClaimsKeepTheirConfigurationKey(t *testing.T) { + // The worker assigns unnamed entries an ID of "manual:
". The broker + // prefixes that entry ID with "manual:" to keep configuration keys separate + // from host UUIDs, so an unnamed entry has two prefixes. + const manualID = "manual:10.0.0.9" + const storeKey = "manual:manual:10.0.0.9" + withUUID := manualToEnriched(manualNodeStatus{ + ID: manualID, + HostUUID: "real-uuid", + }) + require.Equal(t, storeKey, withUUID.storeKey(), "store key") + noUUID := manualToEnriched(manualNodeStatus{ID: manualID}) + require.Equal(t, storeKey, noUUID.storeKey(), "store key") } // TestDiscoveryStoreRejectsEmptyKey verifies a node with no operational key is // dropped rather than silently keyed by name. func TestDiscoveryStore_RejectsEmptyKey(t *testing.T) { s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "no-uuid"}, sourceManual) // HostUUID empty + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: "no-uuid", + }, sourceManual) // HostUUID empty require.Empty(t, s.Snapshot(), "a node with no hostUuid must be dropped") } @@ -63,12 +209,26 @@ func TestDiscoveryStore_RejectsEmptyKey(t *testing.T) { // manual-nodes crash) from evicting a still-live scanner node. func TestDiscoveryStore_PreservesClaimsFromOtherSources(t *testing.T) { const uuid = "shared-uuid" + const name = "host" + const nodeInfoPort = 14318 // Manual-remove overlap: scanner + manual both claim uuid; removing the // manual claim must leave the scanner's node in the snapshot. s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid, Port: 14318}, sourceScanner) - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid, Port: 14318}, sourceManual) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + Port: nodeInfoPort, + }, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + Port: nodeInfoPort, + }, sourceManual) require.Len(t, s.Snapshot(), 1, "shared uuid should be one record") s.Remove(uuid, sourceManual) require.Len(t, s.Snapshot(), 1, "removing the manual claim evicted the live scanner node") @@ -77,8 +237,18 @@ func TestDiscoveryStore_PreservesClaimsFromOtherSources(t *testing.T) { // Symmetric: removing the scanner claim leaves the manual node. s = newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceScanner) - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceManual) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceManual) s.Remove(uuid, sourceScanner) require.Len(t, s.Snapshot(), 1, "removing the scanner claim evicted the manual node") } @@ -88,18 +258,30 @@ func TestDiscoveryStore_PreservesClaimsFromOtherSources(t *testing.T) { // per-engine models the manual probe lacks. func TestDiscoveryStoreScannerProjection_Wins(t *testing.T) { const uuid = "shared-uuid" + const manualName = "manual-host" + const scannerName = "mdns-host" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "manual-host", HostUUID: uuid, Trusted: false}, sourceManual) - s.Upsert(EnrichedNode{ID: "mdns-host", HostUUID: uuid, Trusted: true}, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: manualName, + HostUUID: uuid, + }, sourceManual) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: scannerName, + HostUUID: uuid, + }, sourceScanner) snap := s.Snapshot() require.Len(t, snap, 1, "want one record") - require.Equal(t, "mdns-host", snap[0].ID, "scanner projection should win when both claim") + require.Equal(t, scannerName, snap[0].ID, "scanner projection should win when both claim") require.True(t, snap[0].Trusted, "scanner projection should win when both claim") // Drop the scanner claim: the manual projection takes over. s.Remove(uuid, sourceScanner) snap = s.Snapshot() require.Len(t, snap, 1, "manual projection should surface after scanner removal") - require.Equal(t, "manual-host", snap[0].ID, "manual projection should surface after scanner removal (%v)", snap) + require.Equal(t, manualName, snap[0].ID, "manual projection should surface after scanner removal (%v)", snap) } func newManualTestBroker() *Broker { @@ -112,7 +294,13 @@ func newManualTestBroker() *Broker { } func manualStatus(id, addr, uuid string) manualNodeStatus { - return manualNodeStatus{ID: id, Address: addr, HostUUID: uuid, NodeInfoPort: 14318} + return manualNodeStatus{ + IdentityVerification: noderec.IdentityPinned, + ID: id, + Address: addr, + HostUUID: uuid, + NodeInfoPort: 14318, + } } // TestManualAliasesShareKeyUntilLastRemoved covers both removal orders: two @@ -178,7 +366,12 @@ func TestManualRekey_ReprojectsSharedOldKey(t *testing.T) { func TestDiscoveryStore_IgnoresRemovalFromUnrelatedSource(t *testing.T) { const uuid = "u" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: "host", + HostUUID: uuid, + }, sourceScanner) // manual never claimed it require.False(t, s.Remove(uuid, sourceManual), "removing an unowned source must report false (nothing removed)") assert.Len(t, s.Snapshot(), 1, "removing an unowned source must not drop the record") @@ -188,9 +381,20 @@ func TestDiscoveryStore_IgnoresRemovalFromUnrelatedSource(t *testing.T) { // the FINAL claim, so the node-loss sweep only fires when a node is truly gone. func TestDiscoveryStoreRemove_ReportsFinalClaim(t *testing.T) { const uuid = "shared-uuid" + const name = "host" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceScanner) - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceManual) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceScanner) + s.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceManual) require.False(t, s.Remove(uuid, sourceScanner), "removing one of two claims must report false (node still present)") require.True(t, s.Remove(uuid, sourceManual), "removing the surviving claim must report true (node now gone)") @@ -201,7 +405,13 @@ func TestDiscoveryStoreRemove_ReportsFinalClaim(t *testing.T) { // the given display name and stable host UUID. func nodeRemovedFrame(t *testing.T, name, uuid string) json.RawMessage { t.Helper() - b, err := json.Marshal(noderec.NodeEvent{Node: noderec.DirectoryNode{Name: name, HostUUID: uuid}}) + b, err := json.Marshal(noderec.NodeEvent{ + Node: noderec.DirectoryNode{ + IdentityVerification: noderec.IdentityPinned, + Name: name, + HostUUID: uuid, + }, + }) require.NoError(t, err, "marshal node event") return b } @@ -212,14 +422,28 @@ func nodeRemovedFrame(t *testing.T, name, uuid string) json.RawMessage { // wrongly failed. func TestHandleNotify_ReportsNodeLostOnlyAfterFinalClaimIsRemoved(t *testing.T) { const uuid = "peer-uuid" + const name = "peer-friendly-name" store := newDiscoveryStore() - store.Upsert(EnrichedNode{ID: "peer-friendly-name", HostUUID: uuid}, sourceScanner) - store.Upsert(EnrichedNode{ID: "peer-friendly-name", HostUUID: uuid}, sourceManual) + store.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceScanner) + store.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceManual) var calls int - sp := &scannerProcess{store: store, onNodeLost: func(string, string) { calls++ }} + sp := &scannerProcess{ + store: store, + onNodeLost: func(string, string) { calls++ }, + } - sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, "peer-friendly-name", uuid)) + sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, name, uuid)) require.Equal(t, 0, calls, "onNodeLost fired") require.Len(t, store.Snapshot(), 1, "dual-source node should remain after one source leaves") @@ -232,11 +456,23 @@ func TestHandleNotifyNodeLost_UsesHostUUID(t *testing.T) { const uuid = "peer-uuid" const name = "peer-friendly-name" store := newDiscoveryStore() - store.Upsert(EnrichedNode{ID: name, HostUUID: uuid}, sourceScanner) + store.Upsert(EnrichedNode{ + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + ID: name, + HostUUID: uuid, + }, sourceScanner) var gotUUID, gotName string var calls int - sp := &scannerProcess{store: store, onNodeLost: func(u, n string) { calls++; gotUUID = u; gotName = n }} + sp := &scannerProcess{ + store: store, + onNodeLost: func(u, n string) { + calls++ + gotUUID = u + gotName = n + }, + } sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, name, uuid)) @@ -246,8 +482,9 @@ func TestHandleNotifyNodeLost_UsesHostUUID(t *testing.T) { } func TestScannerProcess_RoutesAndRemovesTelemetry(t *testing.T) { + const hostUUID = "peer-uuid" want := noderec.NodeTelemetry{ - HostUUID: "peer-uuid", + HostUUID: hostUUID, GPUUtilizationPct: 84, TelemetryValid: true, MSSince: 137, @@ -257,7 +494,14 @@ func TestScannerProcess_RoutesAndRemovesTelemetry(t *testing.T) { var got noderec.NodeTelemetry var removed string + store := newDiscoveryStore() + store.Upsert(EnrichedNode{ + HostUUID: hostUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + }, sourceScanner) sp := &scannerProcess{ + store: store, onTelemetry: func(value noderec.NodeTelemetry) { got = value }, onTelemetryRemoved: func(hostUUID string) { removed = hostUUID }, } @@ -267,8 +511,8 @@ func TestScannerProcess_RoutesAndRemovesTelemetry(t *testing.T) { require.Equal(t, want.MSSince, got.MSSince, "routed telemetry (%v, %v)", got, want) require.Equal(t, uint32(84), got.GPUUtilizationPct, "routed utilization") - sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, "peer", "peer-uuid")) - require.Equal(t, "peer-uuid", removed, "removed telemetry host") + sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, "peer", hostUUID)) + require.Equal(t, hostUUID, removed, "removed telemetry host") } func TestDiscoveryStore_SnapshotPreservesScannerIdentityFields(t *testing.T) { diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index 83863167b..9b70497e4 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -26,20 +26,21 @@ import ( // (bridgeManualNode); the rest project into the discovery store via // manualToEnriched. type manualNodeStatus struct { - ID string `json:"id"` - Address string `json:"address"` - OllamaUp bool `json:"ollama_up"` - OllamaPort int `json:"ollama_port"` - OllamaModels []string `json:"ollama_models,omitempty"` - LMStudioUp bool `json:"lmstudio_up"` - LMStudioPort int `json:"lmstudio_port"` - LMStudioModels []string `json:"lmstudio_models,omitempty"` - NodeInfoPort int `json:"node_info_port"` - GPUs []GPUInfo `json:"gpus"` - CPU *CPUInfo `json:"cpu"` - Memory *MemoryInfo `json:"memory"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + ID string `json:"id"` + Address string `json:"address"` + OllamaUp bool `json:"ollama_up"` + OllamaPort int `json:"ollama_port"` + OllamaModels []string `json:"ollama_models,omitempty"` + LMStudioUp bool `json:"lmstudio_up"` + LMStudioPort int `json:"lmstudio_port"` + LMStudioModels []string `json:"lmstudio_models,omitempty"` + NodeInfoPort int `json:"node_info_port"` + GPUs []GPUInfo `json:"gpus"` + CPU *CPUInfo `json:"cpu"` + Memory *MemoryInfo `json:"memory"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` // HostUUID is the remote's stable per-host identity, learned from its // node-info /v1/node-info. It lets a manual node key by the same permanent // identity as mDNS-discovered nodes (and dedup with itself when the same @@ -52,6 +53,13 @@ type manualNodeStatusEntry struct { receivedAt time.Time } +// manualNodeRemoveParams distinguishes a worker entry ID from a broker node key. +// The caller must supply exactly one selector. +type manualNodeRemoveParams struct { + ID string `json:"id,omitempty"` + NodeID string `json:"nodeId,omitempty"` +} + type persistedManualNode struct { ID string `json:"id,omitempty"` Address string `json:"address"` @@ -185,22 +193,18 @@ func (b *Broker) replayPersistedManualNodes(worker *rpcWorker) { } } -func (b *Broker) manualAliasForKey(id string) string { +// manualAliasForKey resolves only broker node keys. A matching entry ID cannot +// take precedence over another entry's node key. No match returns an empty ID. +func (b *Broker) manualAliasForKey(key string) string { b.manualMu.Lock() defer b.manualMu.Unlock() - if _, ok := b.manualNodeKeys[id]; ok { - return id - } best := "" - for alias, key := range b.manualNodeKeys { - if key == id && (best == "" || alias < best) { + for alias, nodeKey := range b.manualNodeKeys { + if nodeKey == key && (best == "" || alias < best) { best = alias } } - if best != "" { - return best - } - return id + return best } func manualNodeTelemetry(status manualNodeStatus, hostUUID string) noderec.NodeTelemetry { @@ -221,34 +225,38 @@ func manualNodeTelemetry(status manualNodeStatus, hostUUID string) noderec.NodeT // manualToEnriched projects a manual node's status onto the EnrichedNode // the discovery store holds, so manual nodes share the single // discovery:get-nodes / discovery:nodes-changed snapshot with mDNS nodes. -// Port is the node-info HTTP port the prober reached (mirroring the SRV +// Port is the node-info HTTPS port the prober attempted (mirroring the SRV // port the scanner records for mDNS nodes); the user-entered address // becomes the node's host/address so the snapshot's ipAddress projection // can resolve it when the address is an IP literal. func manualToEnriched(s manualNodeStatus) EnrichedNode { - // Every node carries a non-empty operational key by the time it reaches the - // store: the remote's real hostUuid once node-info reports it, else the - // manual id (the user's name or "manual:
") until then. This is the - // manual-node ingestion boundary — downstream keys off HostUUID with no - // fallback. Once the real UUID is learned, a manually-added machine - // that's also discovered over mDNS collapses to the one hostUuid-keyed entry. + // Only a pinned response may join a canonical host. The explicit manual + // engine target otherwise uses a separate key shared with its proxy route. + // Treat the worker's ID as opaque: an unnamed "1.2.3.4" entry already has + // ID "manual:1.2.3.4", so our namespace produces "manual:manual:1.2.3.4". + // Stripping that inner prefix would collide with a named "1.2.3.4" entry, + // whose worker ID is "1.2.3.4" and whose broker key is "manual:1.2.3.4". hostUUID := s.HostUUID - if hostUUID == "" { - hostUUID = s.ID + if hostUUID == "" || s.IdentityVerification != noderec.IdentityPinned { + hostUUID = "manual:" + s.ID } en := EnrichedNode{ DiscoveryRecordID: "manual:" + s.ID, - IdentityVerification: noderec.IdentityUnverified, + IdentityVerification: s.IdentityVerification, ID: s.ID, HostUUID: hostUUID, Host: s.Address, Port: s.NodeInfoPort, - GPUs: s.GPUs, - CPU: s.CPU, - Memory: s.Memory, Models: mergeModels(s.OllamaModels, s.LMStudioModels), ModelsByEngine: manualModelsByEngine(s), } + if s.IdentityVerification == noderec.IdentityPinned { + en.GPUs, en.CPU, en.Memory = s.GPUs, s.CPU, s.Memory + en.Trusted = true + en.Clustered = true + } else { + en.IdentityVerification = noderec.IdentityUnverified + } if s.Address != "" { en.Addresses = []string{s.Address} } diff --git a/services/nvpair-ui-broker/manualnodes_persistence_test.go b/services/nvpair-ui-broker/manualnodes_persistence_test.go index d14e4d0a7..21bffcb6c 100644 --- a/services/nvpair-ui-broker/manualnodes_persistence_test.go +++ b/services/nvpair-ui-broker/manualnodes_persistence_test.go @@ -68,12 +68,6 @@ func TestReplayPersistedManualNodes_ResendsSavedNodesToWorker(t *testing.T) { } } -func TestManualAliasForKey_PrefersMatchingAliasAndFallsBackDeterministically(t *testing.T) { - b := &Broker{manualNodeKeys: map[string]string{"second": "node-uuid", "first": "node-uuid"}} - assert.Equal(t, "first", b.manualAliasForKey("node-uuid")) - assert.Equal(t, "second", b.manualAliasForKey("second")) -} - func TestManualNodeRelay_PersistsSuccessfulAdd(t *testing.T) { path := filepath.Join(t.TempDir(), "configs", "manual-nodes.json") clientBroker, clientSide := net.Pipe() @@ -114,7 +108,7 @@ func TestManualNodeRelay_PersistsSuccessfulAdd(t *testing.T) { assert.Equal(t, "node.local", entries[0].Address) } -func TestManualNodeRelay_RemovesByOperationalKey(t *testing.T) { +func TestManualNodeRelay_RemovesByNodeKey(t *testing.T) { path := filepath.Join(t.TempDir(), "configs", "manual-nodes.json") require.NoError(t, writePersistedManualNodes(path, []persistedManualNode{{ID: "lab", Address: "node.local", Name: "lab"}})) clientBroker, clientSide := net.Pipe() @@ -152,7 +146,7 @@ func TestManualNodeRelay_RemovesByOperationalKey(t *testing.T) { JSONRPC: "2.0", ID: &id, Method: "node/remove", - Params: json.RawMessage(`{"id":"node-uuid"}`), + Params: json.RawMessage(`{"nodeId":"node-uuid"}`), }) response, err := NewCodec(clientSide).Read() require.NoError(t, err, "read remove response") diff --git a/services/nvpair-ui-broker/manualnodes_test.go b/services/nvpair-ui-broker/manualnodes_test.go new file mode 100644 index 000000000..4fe036f2b --- /dev/null +++ b/services/nvpair-ui-broker/manualnodes_test.go @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "net" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "nvpair-shared/noderec" +) + +// Two entries deliberately overlap: "lab" has broker key "manual:lab", while +// "manual:lab" is a different entry with broker key "manual:manual:lab". Each +// case sends a removal through the broker to a fake worker that acknowledges it. +// The assertions check that the worker receives the intended exact entry ID +// and that persistence removes only that entry, leaving the other one intact. +// This covers both the desktop's nodeId selector and the TUI's id selector. +func TestManualNodeRelay_DistinguishesEntryIDsFromNodeKeys(t *testing.T) { + const entryID = "lab" + const prefixedEntryID = "manual:lab" + const prefixedNodeID = "manual:manual:lab" + type removalRequest struct { + ID string `json:"id,omitempty"` + NodeID string `json:"nodeId,omitempty"` + } + type workerObservation struct { + request removalRequest + err error + } + test := func(name string, request removalRequest, wantRemovedID, wantRemainingID string) { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "manual-nodes.json") + require.NoError(t, writePersistedManualNodes(path, []persistedManualNode{ + {ID: entryID, Name: entryID, Address: "first.local"}, + {ID: prefixedEntryID, Name: prefixedEntryID, Address: "second.local"}, + })) + clientBroker, clientSide := net.Pipe() + workerBroker, workerSide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, clientBroker.Close()) + assert.NoError(t, clientSide.Close()) + assert.NoError(t, workerBroker.Close()) + assert.NoError(t, workerSide.Close()) + }) + require.NoError(t, clientSide.SetReadDeadline(time.Now().Add(2*time.Second))) + require.NoError(t, workerSide.SetReadDeadline(time.Now().Add(2*time.Second))) + worker := &rpcWorker{peer: NewPeer(NewCodec(workerBroker))} + go worker.peer.Serve(nil, nil) + observed := make(chan workerObservation, 1) + go func() { + codec := NewCodec(workerSide) + message, err := codec.Read() + if err != nil { + observed <- workerObservation{err: err} + return + } + var forwarded removalRequest + if err := json.Unmarshal(message.Params, &forwarded); err != nil { + observed <- workerObservation{err: err} + return + } + err = codec.Respond(message.ID, map[string]bool{"removed": true}) + observed <- workerObservation{request: forwarded, err: err} + }() + broker := &Broker{ + codec: NewCodec(clientBroker), + manualNodesConfigPath: path, + manualNodeKeys: map[string]string{ + entryID: prefixedEntryID, + prefixedEntryID: prefixedNodeID, + }, + } + broker.setManualNodes(worker) + params, err := json.Marshal(request) + require.NoError(t, err) + id := json.RawMessage("1") + + go broker.relayToManualNodes(&Message{ + JSONRPC: "2.0", + ID: &id, + Method: "node/remove", + Params: params, + }) + response, err := NewCodec(clientSide).Read() + require.NoError(t, err) + require.Nil(t, response.Error) + observation := <-observed + require.NoError(t, observation.err) + assert.Equal(t, wantRemovedID, observation.request.ID) + assert.Empty(t, observation.request.NodeID, "the worker receives only its entry ID") + entries, err := loadPersistedManualNodes(path) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, wantRemainingID, entries[0].ID) + }) + } + test("node key overlaps another entry ID", removalRequest{NodeID: prefixedEntryID}, entryID, prefixedEntryID) + test("prefixed node key", removalRequest{NodeID: prefixedNodeID}, prefixedEntryID, entryID) + test("entry ID overlaps another node key", removalRequest{ID: prefixedEntryID}, prefixedEntryID, entryID) + test("plain entry ID", removalRequest{ID: entryID}, entryID, prefixedEntryID) +} + +func TestManualNodeRelay_LeavesUnknownNodeKeysUnchanged(t *testing.T) { + const entryID = "lab" + path := filepath.Join(t.TempDir(), "manual-nodes.json") + entries := []persistedManualNode{ + {ID: entryID, Name: entryID, Address: "node.local"}, + } + require.NoError(t, writePersistedManualNodes(path, entries)) + var clientMessages bytes.Buffer + var workerRequests bytes.Buffer + broker := &Broker{ + codec: NewCodec(&clientMessages), + manualNodesConfigPath: path, + manualNodeKeys: map[string]string{entryID: "manual:lab"}, + } + broker.setManualNodes(&rpcWorker{peer: NewPeer(NewCodec(&workerRequests))}) + id := json.RawMessage("1") + + // This is an existing entry ID, but it is not a broker node key. + broker.relayToManualNodes(&Message{ + JSONRPC: "2.0", + ID: &id, + Method: "node/remove", + Params: json.RawMessage(`{"nodeId":"lab"}`), + }) + + response, err := broker.codec.Read() + require.NoError(t, err) + require.Nil(t, response.Error) + type removalResult struct { + Removed bool `json:"removed"` + } + var result removalResult + require.NoError(t, json.Unmarshal(response.Result, &result)) + assert.False(t, result.Removed) + assert.Empty(t, workerRequests.String(), "an unknown node key must not reach the worker") + saved, err := loadPersistedManualNodes(path) + require.NoError(t, err) + assert.Equal(t, entries, saved) +} + +func TestManualNodeRelay_RejectsAmbiguousOrMissingRemovalSelectors(t *testing.T) { + test := func(name, params string) { + t.Run(name, func(t *testing.T) { + var clientMessages bytes.Buffer + var workerRequests bytes.Buffer + broker := &Broker{codec: NewCodec(&clientMessages)} + broker.setManualNodes(&rpcWorker{peer: NewPeer(NewCodec(&workerRequests))}) + id := json.RawMessage("1") + + broker.relayToManualNodes(&Message{ + JSONRPC: "2.0", + ID: &id, + Method: "node/remove", + Params: json.RawMessage(params), + }) + + response, err := broker.codec.Read() + require.NoError(t, err) + require.NotNil(t, response.Error) + assert.Equal(t, -32602, response.Error.Code) + assert.Empty(t, workerRequests.String(), "invalid selectors must not reach the worker") + }) + } + test("both selectors", `{"id":"lab","nodeId":"manual:lab"}`) + test("missing selectors", `{}`) + test("empty entry ID", `{"id":""}`) + test("empty node key", `{"nodeId":""}`) + test("null parameters", `null`) + test("non-string selector", `{"id":123}`) +} + +func TestManualAliasForKey_ResolvesOnlyNodeKeys(t *testing.T) { + const hostUUID = "node-uuid" + const entryID = "lab" + const prefixedEntryID = "manual:lab" + const prefixedNodeID = "manual:manual:lab" + broker := &Broker{ + manualNodeKeys: map[string]string{ + "second": hostUUID, + "first": hostUUID, + entryID: prefixedEntryID, + prefixedEntryID: prefixedNodeID, + }, + } + test := func(name, key, wantID string) { + t.Run(name, func(t *testing.T) { + assert.Equal(t, wantID, broker.manualAliasForKey(key)) + }) + } + test("shared node key resolves deterministically", hostUUID, "first") + test("node key overlaps another entry ID", prefixedEntryID, entryID) + test("prefixed node key", prefixedNodeID, prefixedEntryID) + test("entry ID is not a node key", entryID, "") + test("unknown node key", "missing", "") +} + +func TestManualToEnriched_ExcludesUnverifiedHardwareFromServiceRecords(t *testing.T) { + const modelName = "explicit-engine-model" + status := manualNodeStatus{ + ID: "configured", + HostUUID: "claimed-peer", + IdentityVerification: noderec.IdentityUnverified, + GPUs: []GPUInfo{{Name: "public GPU"}}, + CPU: &CPUInfo{Name: "public CPU"}, + Memory: &MemoryInfo{TotalBytes: 1024}, + OllamaModels: []string{modelName}, + } + + node := manualToEnriched(status) + + assert.Empty(t, node.GPUs) + assert.Nil(t, node.CPU) + assert.Nil(t, node.Memory) + assert.Equal(t, "manual:configured", node.HostUUID) + assert.Equal(t, []string{modelName}, node.Models) +} diff --git a/services/nvpair-ui-broker/relay/relay.go b/services/nvpair-ui-broker/relay/relay.go index ed02240cc..6397afa99 100644 --- a/services/nvpair-ui-broker/relay/relay.go +++ b/services/nvpair-ui-broker/relay/relay.go @@ -102,7 +102,7 @@ type Subscriber struct { sendMu sync.Mutex } -// Directory is the broker's view of all LAN nodes (keyed by hostUuid) plus its +// Directory is the broker's view of all LAN nodes (keyed by discoveryRecordId) plus its // subscriber set. It's fed by the daemon's node-* events via Apply and queried // by discovery:get-nodes via Snapshot. type Directory struct { @@ -156,7 +156,7 @@ func (d *Directory) filteredLocked(f noderec.SubscribeParams) []noderec.Director out = append(out, n) } } - sort.Slice(out, func(i, j int) bool { return out[i].HostUUID < out[j].HostUUID }) + sort.Slice(out, func(i, j int) bool { return out[i].DiscoveryRecordID < out[j].DiscoveryRecordID }) return out } @@ -167,20 +167,24 @@ func (d *Directory) Unsubscribe(id int) { d.mu.Unlock() } -// Apply folds a daemon node-* delta into the directory, then re-sends every -// subscriber its full filtered snapshot. method (one of noderec.NotifyNode{ -// Discovered,Updated,Removed}) only updates the directory — removed drops the -// node by hostUuid, discovered/updated upsert it. Every subscriber is re-pushed -// on any change (not just those matching the changed node) so each consumer's -// set is always the authoritative current list; the push is idempotent (the -// consumer replaces with the same set), so a re-push for an unrelated change is a -// cheap no-op at this scale. +// Apply adds, replaces, or removes one scanner record by DiscoveryRecordID. +// HostUUID is only a claim in an unverified record. Records with the same +// HostUUID must stay independent, so removing an unverified discovery cannot +// remove the verified peer's record. +// +// It then sends every subscriber the complete set matching that subscriber's +// filter. Subscribers replace their previous set rather than applying changes +// to individual records. Refreshing every subscriber also removes records that +// no longer match after an update. func (d *Directory) Apply(method string, node noderec.DirectoryNode) { + if node.HostUUID == "" || node.DiscoveryRecordID == "" { + return + } d.mu.Lock() if method == noderec.NotifyNodeRemoved { - delete(d.nodes, node.HostUUID) + delete(d.nodes, node.DiscoveryRecordID) } else { - d.nodes[node.HostUUID] = node + d.nodes[node.DiscoveryRecordID] = node } subs := make([]*Subscriber, 0, len(d.subs)) for _, s := range d.subs { @@ -196,7 +200,7 @@ func (d *Directory) Apply(method string, node noderec.DirectoryNode) { } // Snapshot returns the directory optionally filtered to one service, sorted by -// hostUuid. +// DiscoveryRecordID. func (d *Directory) Snapshot(filter noderec.ServiceKey) []noderec.DirectoryNode { d.mu.Lock() out := make([]noderec.DirectoryNode, 0, len(d.nodes)) @@ -207,6 +211,6 @@ func (d *Directory) Snapshot(filter noderec.ServiceKey) []noderec.DirectoryNode out = append(out, n) } d.mu.Unlock() - sort.Slice(out, func(i, j int) bool { return out[i].HostUUID < out[j].HostUUID }) + sort.Slice(out, func(i, j int) bool { return out[i].DiscoveryRecordID < out[j].DiscoveryRecordID }) return out } diff --git a/services/nvpair-ui-broker/relay/relay_test.go b/services/nvpair-ui-broker/relay/relay_test.go index 95b76e531..2bf64a005 100644 --- a/services/nvpair-ui-broker/relay/relay_test.go +++ b/services/nvpair-ui-broker/relay/relay_test.go @@ -12,6 +12,30 @@ import ( "nvpair-shared/noderec" ) +// Add a verified peer and a candidate copying its host UUID, then remove only +// the candidate. The service-filtered snapshot must still contain the original +// peer, proving that candidate events do not overwrite or remove its target. +func TestDirectory_CandidateClaimsCannotReplaceServiceTargets(t *testing.T) { + directory := NewDirectory() + peer := olNode("host-a") + peer.DiscoveryRecordID = "peer:host-a" + peer.IdentityVerification = noderec.IdentityPinned + peer.Trusted = true + peer.IP = "192.0.2.20" + + candidate := olNode("host-a") + candidate.DiscoveryRecordID = "candidate:rogue" + candidate.IdentityVerification = noderec.IdentityUnverified + candidate.Trusted = false + candidate.IP = "192.0.2.30" + + directory.Apply(noderec.NotifyNodeDiscovered, peer) + directory.Apply(noderec.NotifyNodeDiscovered, candidate) + directory.Apply(noderec.NotifyNodeRemoved, candidate) + + require.Equal(t, []noderec.DirectoryNode{peer}, directory.Snapshot(noderec.ServiceOllama)) +} + func TestRegistrationCache_StoresAndReplacesServiceRegistrations(t *testing.T) { c := NewRegistrationCache() require.True(t, c.Register(noderec.RegisterParams{Service: noderec.ServiceNodeInfo, Port: 14318}), "first register should change") @@ -59,10 +83,24 @@ func ids(nodes []noderec.DirectoryNode) []string { } func olNode(id string) noderec.DirectoryNode { - return noderec.DirectoryNode{HostUUID: id, Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}} + return noderec.DirectoryNode{ + DiscoveryRecordID: "local:" + id, + IdentityVerification: noderec.IdentityLocal, + HostUUID: id, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceOllama: {Port: 11434}, + }, + } } func erNode(id string) noderec.DirectoryNode { - return noderec.DirectoryNode{HostUUID: id, Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceErrors: {Port: 14319}}} + return noderec.DirectoryNode{ + DiscoveryRecordID: "local:" + id, + IdentityVerification: noderec.IdentityLocal, + HostUUID: id, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceErrors: {Port: 14319}, + }, + } } func TestDirectorySubscribe_SendsInitialSnapshot(t *testing.T) { diff --git a/services/nvpair-ui-broker/scheduler_test.go b/services/nvpair-ui-broker/scheduler_test.go index 58c5af4e3..ffcc618ac 100644 --- a/services/nvpair-ui-broker/scheduler_test.go +++ b/services/nvpair-ui-broker/scheduler_test.go @@ -38,7 +38,7 @@ func TestSchedulerFeedBaseline_PrecedesConcurrentLiveWorkload(t *testing.T) { HostUUID: "a", GPUUtilizationPct: 50, TelemetryValid: true, - }, time.Now()) + }, time.Now(), time.Now()) feedLocked := make(chan struct{}) initDone := make(chan int, 1) diff --git a/services/nvpair-ui-broker/telemetry.go b/services/nvpair-ui-broker/telemetry.go index 7e96b9007..a221daed6 100644 --- a/services/nvpair-ui-broker/telemetry.go +++ b/services/nvpair-ui-broker/telemetry.go @@ -66,7 +66,7 @@ func newTelemetryCache() *telemetryCache { // Upsert records one source's latest observation and returns the source-aware // projection to forward. Scanner is authoritative while present; manual // telemetry is the fallback for nodes without a scanner claim. -func (c *telemetryCache) Upsert(source nodeSource, value noderec.NodeTelemetry, receivedAt time.Time) (noderec.NodeTelemetry, bool) { +func (c *telemetryCache) Upsert(source nodeSource, value noderec.NodeTelemetry, receivedAt, now time.Time) (noderec.NodeTelemetry, bool) { if value.HostUUID == "" { return noderec.NodeTelemetry{}, false } @@ -84,7 +84,7 @@ func (c *telemetryCache) Upsert(source nodeSource, value noderec.NodeTelemetry, if !ok { return noderec.NodeTelemetry{}, false } - return observedTelemetryAt(*projected, receivedAt), true + return observedTelemetryAt(*projected, now), true } // Remove drops one source. If another source survives, its current projection @@ -136,12 +136,13 @@ func (b *Broker) ingestTelemetry(source nodeSource, value noderec.NodeTelemetry) } func (b *Broker) ingestTelemetryAt(source nodeSource, value noderec.NodeTelemetry, receivedAt time.Time) { - if b.telemetry == nil { + if b.telemetry == nil || b.store == nil || !b.store.verifiedSource(value.HostUUID, source) { return } - projected, ok := b.telemetry.Upsert(source, value, receivedAt) + projectedAt := time.Now() + projected, ok := b.telemetry.Upsert(source, value, receivedAt, projectedAt) if ok { - b.fanTelemetryToScheduler(projected) + b.fanTelemetryToScheduler(projected, projectedAt) } } @@ -149,13 +150,16 @@ func (b *Broker) removeTelemetry(source nodeSource, hostUUID string) { if b.telemetry == nil { return } - projected, ok := b.telemetry.Remove(hostUUID, source, time.Now()) + projectedAt := time.Now() + projected, ok := b.telemetry.Remove(hostUUID, source, projectedAt) if ok { - b.fanTelemetryToScheduler(projected) + b.fanTelemetryToScheduler(projected, projectedAt) } } -func (b *Broker) fanTelemetryToScheduler(telemetry noderec.NodeTelemetry) { +// fanTelemetryToScheduler includes time waiting for the ordered scheduler feed +// and live authorization checks after the cache projected this sample. +func (b *Broker) fanTelemetryToScheduler(telemetry noderec.NodeTelemetry, projectedAt time.Time) { b.schedulerFeedMu.Lock() defer b.schedulerFeedMu.Unlock() @@ -163,6 +167,7 @@ func (b *Broker) fanTelemetryToScheduler(telemetry noderec.NodeTelemetry) { if scheduler == nil { return } + telemetry = observedTelemetryAt(telemetryObservation{value: telemetry, receivedAt: projectedAt}, time.Now()) if err := scheduler.Notify(schedulerwire.MethodTelemetry, telemetry); err != nil { slog.Warn("fan telemetry to scheduler failed", "host_uuid", telemetry.HostUUID, "err", err) } diff --git a/services/nvpair-ui-broker/telemetry_test.go b/services/nvpair-ui-broker/telemetry_test.go index 60b3eb496..cf19adeae 100644 --- a/services/nvpair-ui-broker/telemetry_test.go +++ b/services/nvpair-ui-broker/telemetry_test.go @@ -4,11 +4,13 @@ package main import ( + "bytes" "encoding/json" "net" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "nvpair-shared/noderec" @@ -17,14 +19,14 @@ import ( func TestTelemetryCache_IncludesElapsedTimeInObservationAge(t *testing.T) { cache := newTelemetryCache() - receivedAt := time.Unix(1_700_000_000, 0) + receivedAt := time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant input := noderec.NodeTelemetry{ HostUUID: "node-a", GPUUtilizationPct: 84, TelemetryValid: true, MSSince: 137, } - projected, ok := cache.Upsert(sourceScanner, input, receivedAt) + projected, ok := cache.Upsert(sourceScanner, input, receivedAt, receivedAt) require.True(t, ok, "initial projection (%v)", projected) require.Equal(t, int64(137), projected.MSSince, "initial projection (%v)", projected) snapshot := cache.Snapshot(receivedAt.Add(250 * time.Millisecond)) @@ -33,61 +35,96 @@ func TestTelemetryCache_IncludesElapsedTimeInObservationAge(t *testing.T) { require.Equal(t, uint32(84), snapshot[0].GPUUtilizationPct, "cached utilization") } +// Cache a verified scanner sample, then add an unverified manual entry claiming +// the same host with a low GPU reading. Removing the scanner sample must leave +// no scheduling telemetry, while both entries remain visible in discovery. +func TestManualTelemetry_PublicClaimsCannotBecomeSchedulingFallback(t *testing.T) { + const peerUUID = "peer" + broker := newManualTestBroker() + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + }, sourceScanner) + broker.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{ + HostUUID: peerUUID, + GPUUtilizationPct: 80, + TelemetryValid: true, + }) + broker.upsertManualNode(manualNodeStatus{ + ID: "configured", + HostUUID: peerUUID, + Address: "192.0.2.30", + IdentityVerification: noderec.IdentityUnverified, + TelemetryValid: true, + GPUs: []GPUInfo{{UtilizationPercent: 1}}, + }) + + broker.removeTelemetry(sourceScanner, peerUUID) + + require.Empty(t, broker.telemetry.Snapshot(time.Now())) + require.Len(t, broker.store.Snapshot(), 2) +} + func TestTelemetryCache_PrefersScannerAndFallsBackToManual(t *testing.T) { + const hostUUID = "node-a" cache := newTelemetryCache() - now := time.Unix(1_700_000_000, 0) + now := time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant scanner := noderec.NodeTelemetry{ - HostUUID: "node-a", + HostUUID: hostUUID, GPUUtilizationPct: 20, TelemetryValid: true, } manual := noderec.NodeTelemetry{ - HostUUID: "node-a", + HostUUID: hostUUID, GPUUtilizationPct: 70, TelemetryValid: true, } - cache.Upsert(sourceScanner, scanner, now) - projected, ok := cache.Upsert(sourceManual, manual, now.Add(time.Second)) + cache.Upsert(sourceScanner, scanner, now, now) + projected, ok := cache.Upsert(sourceManual, manual, now.Add(time.Second), now.Add(time.Second)) require.True(t, ok, "scanner projection (%v)", projected) require.Equal(t, uint32(20), projected.GPUUtilizationPct, "scanner projection (%v)", projected) - projected, ok = cache.Remove("node-a", sourceScanner, now.Add(2*time.Second)) + projected, ok = cache.Remove(hostUUID, sourceScanner, now.Add(2*time.Second)) require.True(t, ok, "manual fallback (%v)", projected) require.Equal(t, uint32(70), projected.GPUUtilizationPct, "manual fallback (%v)", projected) require.Equal(t, int64(1_000), projected.MSSince, "manual fallback (%v)", projected) - projected, ok = cache.Remove("node-a", sourceManual, now.Add(3*time.Second)) + projected, ok = cache.Remove(hostUUID, sourceManual, now.Add(3*time.Second)) require.True(t, ok, "final removal projection (%v)", projected) - require.Equal(t, "node-a", projected.HostUUID, "final removal projection (%v)", projected) + require.Equal(t, hostUUID, projected.HostUUID, "final removal projection (%v)", projected) require.False(t, projected.TelemetryValid, "final removal projection (%v)", projected) require.Equal(t, int64(0), projected.MSSince, "final removal projection (%v)", projected) require.Empty(t, cache.Snapshot(now.Add(4*time.Second)), "cache retained final removal") } func TestTelemetryCacheSnapshot_IsSortedAndNormalizesInvalidAge(t *testing.T) { + const validUUID = "node-a" + const invalidUUID = "node-z" cache := newTelemetryCache() - now := time.Unix(1_700_000_000, 0) + now := time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant cache.Upsert(sourceScanner, noderec.NodeTelemetry{ - HostUUID: "node-z", + HostUUID: invalidUUID, TelemetryValid: false, MSSince: 999, - }, now) + }, now, now) cache.Upsert(sourceScanner, noderec.NodeTelemetry{ - HostUUID: "node-a", + HostUUID: validUUID, TelemetryValid: true, MSSince: -50, - }, now) + }, now, now) got := cache.Snapshot(now) require.Len(t, got, 2, "snapshot order") - require.Equal(t, "node-a", got[0].HostUUID, "snapshot order (%v)", got) - require.Equal(t, "node-z", got[1].HostUUID, "snapshot order (%v)", got) + require.Equal(t, validUUID, got[0].HostUUID, "snapshot order (%v)", got) + require.Equal(t, invalidUUID, got[1].HostUUID, "snapshot order (%v)", got) require.Equal(t, int64(0), got[0].MSSince, "normalized age") require.Equal(t, int64(0), got[1].MSSince, "normalized age") } func TestReplayTelemetryToScheduler_IncludesCurrentAge(t *testing.T) { + const hostUUID = "node-a" brokerSide, schedulerSide := net.Pipe() t.Cleanup(func() { _ = brokerSide.Close() @@ -96,11 +133,11 @@ func TestReplayTelemetryToScheduler_IncludesCurrentAge(t *testing.T) { worker := &rpcWorker{peer: NewPeer(NewCodec(brokerSide))} b := &Broker{telemetry: newTelemetryCache()} b.telemetry.Upsert(sourceScanner, noderec.NodeTelemetry{ - HostUUID: "node-a", + HostUUID: hostUUID, GPUUtilizationPct: 45, TelemetryValid: true, MSSince: 100, - }, time.Now().Add(-250*time.Millisecond)) + }, time.Now().Add(-250*time.Millisecond), time.Now()) replayed := make(chan int, 1) go func() { replayed <- b.replayTelemetryToScheduler(worker) }() @@ -109,11 +146,48 @@ func TestReplayTelemetryToScheduler_IncludesCurrentAge(t *testing.T) { require.Equal(t, schedulerwire.MethodTelemetry, message.Method, "replay method") var got noderec.NodeTelemetry require.NoError(t, json.Unmarshal(message.Params, &got), "decode replay") - require.Equal(t, "node-a", got.HostUUID, "replayed telemetry (%v)", got) + require.Equal(t, hostUUID, got.HostUUID, "replayed telemetry (%v)", got) require.GreaterOrEqual(t, got.MSSince, int64(350), "replayed telemetry (%v)", got) require.Equal(t, 1, <-replayed, "replayed count") } +// Send a verified sample whose recorded arrival was 250 ms ago through the +// live broker-to-scheduler pipe. Its 100 ms source age must become at least +// 350 ms on the wire, preserving time already spent waiting in the broker. +func TestIngestTelemetry_IncludesTimeWaitingInBroker(t *testing.T) { + const peerUUID = "peer" + brokerSide, schedulerSide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, brokerSide.Close()) + assert.NoError(t, schedulerSide.Close()) + }) + broker := newManualTestBroker() + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + }, sourceScanner) + broker.setScheduler(&rpcWorker{peer: NewPeer(NewCodec(brokerSide))}) + // Let the unbuffered pipe's writer and reader run together, then wait for + // ingestion to return before cleanup closes either end of the connection. + done := make(chan struct{}) + go func() { + broker.ingestTelemetryAt(sourceScanner, noderec.NodeTelemetry{ + HostUUID: peerUUID, + TelemetryValid: true, + MSSince: 100, + }, time.Now().Add(-250*time.Millisecond)) + close(done) + }() + require.NoError(t, schedulerSide.SetReadDeadline(time.Now().Add(2*time.Second))) + + message := readSchedulerTestMessage(t, NewCodec(schedulerSide)) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.GreaterOrEqual(t, sample.MSSince, int64(350)) + <-done +} + func TestManualTelemetry_ReprojectsSurvivingAliasAtOriginalAge(t *testing.T) { b := newManualTestBroker() first := manualStatus("first", "10.0.0.1", "node-a") @@ -143,3 +217,45 @@ func TestManualTelemetry_ReprojectsSurvivingAliasAtOriginalAge(t *testing.T) { b.removeManualNode("first") require.Empty(t, b.telemetry.Snapshot(time.Now()), "final manual alias left telemetry cached") } + +func TestLiveTelemetry_IncludesTimeWaitingForSchedulerFeed(t *testing.T) { + const nodeUUID = "local-node" + const sourceAgeMillis = int64(137) + const feedDelay = 100 * time.Millisecond + broker := newManualTestBroker() + broker.store.Upsert(EnrichedNode{HostUUID: nodeUUID, IdentityVerification: noderec.IdentityLocal}, sourceScanner) + var output bytes.Buffer + broker.setScheduler(&rpcWorker{peer: NewPeer(NewCodec(&output))}) + broker.schedulerFeedMu.Lock() + feedLocked := true + done := make(chan struct{}) + t.Cleanup(func() { + if feedLocked { + broker.schedulerFeedMu.Unlock() + } + <-done + }) + go func() { + defer close(done) + broker.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{ + HostUUID: nodeUUID, TelemetryValid: true, MSSince: sourceAgeMillis, + }) + }() + // Cache insertion precedes the feed lock. Hold the sample behind a replay + // long enough to distinguish send-time age from its pre-lock projection. + require.Eventually(t, func() bool { + return len(broker.telemetry.Snapshot(time.Now())) == 1 + }, 2*time.Second, time.Millisecond) + time.Sleep(feedDelay) + broker.schedulerFeedMu.Unlock() + feedLocked = false + <-done + + message, err := NewCodec(&output).Read() + require.NoError(t, err) + require.Equal(t, schedulerwire.MethodTelemetry, message.Method) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.True(t, sample.TelemetryValid) + assert.GreaterOrEqual(t, sample.MSSince, sourceAgeMillis+feedDelay.Milliseconds()) +} diff --git a/services/nvpair-workload-manager/relaysource.go b/services/nvpair-workload-manager/relaysource.go index c1510990b..1de97d716 100644 --- a/services/nvpair-workload-manager/relaysource.go +++ b/services/nvpair-workload-manager/relaysource.go @@ -87,6 +87,9 @@ func (r *relayPeerSource) set(nodes []noderec.DirectoryNode) { // dialed by its ip=/Addresses, not its ID. cluster-uuid= is reconstructed into // TXT so the broadcaster's mTLS pin lookup is unchanged. func directoryToPeerNode(n noderec.DirectoryNode) (PeerNode, bool) { + if n.IdentityVerification != noderec.IdentityPinned || !n.Trusted { + return PeerNode{}, false + } svc, ok := n.Services[noderec.ServiceWorkload] addresses := n.CandidateIPs() if !ok || len(addresses) == 0 { diff --git a/services/nvpair-workload-manager/relaysource_test.go b/services/nvpair-workload-manager/relaysource_test.go index 04e49cdcc..62a342ca1 100644 --- a/services/nvpair-workload-manager/relaysource_test.go +++ b/services/nvpair-workload-manager/relaysource_test.go @@ -13,6 +13,61 @@ import ( "nvpair-shared/noderec" ) +// Feed a verified workload peer and an unverified copy of its UUID into the +// same snapshot. Only the verified endpoint may become a synchronization target; +// the later candidate must neither replace it nor add another target. +func TestRelayPeerSource_UnverifiedClaimsCannotReplaceVerifiedEndpoints(t *testing.T) { + const peerAddress = "192.0.2.20" + source := newRelayPeerSource("self") + peer := noderec.DirectoryNode{ + HostUUID: "peer", + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + IP: peerAddress, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceWorkload: {Port: 14320}, + }, + } + candidate := peer + candidate.IdentityVerification = noderec.IdentityUnverified + candidate.IP = "192.0.2.30" + + source.set([]noderec.DirectoryNode{peer, candidate}) + + peers, err := source.Nodes(context.Background()) + require.NoError(t, err) + require.Len(t, peers, 1) + assert.Equal(t, []string{peerAddress}, peers[0].Addresses) +} + +func TestRelayPeerSource_RemovesPeersWhenTrustIsWithdrawn(t *testing.T) { + const peerUUID = "peer" + const peerAddress = "192.0.2.20" + source := newRelayPeerSource("self") + peer := noderec.DirectoryNode{ + HostUUID: peerUUID, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + IP: peerAddress, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceWorkload: {Port: 14320}, + }, + } + source.set([]noderec.DirectoryNode{peer}) + peers, err := source.Nodes(context.Background()) + require.NoError(t, err) + require.Len(t, peers, 1) + assert.Equal(t, peerUUID, peers[0].ID) + assert.Equal(t, []string{peerAddress}, peers[0].Addresses) + + peer.Trusted = false + source.set([]noderec.DirectoryNode{peer}) + + peers, err = source.Nodes(context.Background()) + require.NoError(t, err) + assert.Empty(t, peers) +} + // TestSameNameDistinctUUIDPeersSurviveReplace: two peers // that share a hostname but hold distinct hostUuids must remain two broadcast // targets. relayPeerSource keys its map by hostUuid, but peerSet.Replace re-keys @@ -22,8 +77,15 @@ func TestSameNameDistinctUUIDPeers_SurviveReplace(t *testing.T) { src := newRelayPeerSource("self-uuid") node := func(uuid, ip string) noderec.DirectoryNode { return noderec.DirectoryNode{ - HostUUID: uuid, Name: "samehost", IP: ip, IPs: []string{ip, "192.168.1." + ip[len(ip)-1:]}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceWorkload: {Port: 14320}}, + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: uuid, + Name: "samehost", + IP: ip, + IPs: []string{ip, "192.168.1." + ip[len(ip)-1:]}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceWorkload: {Port: 14320}, + }, } } src.set([]noderec.DirectoryNode{node("uuid-1", "10.0.0.1"), node("uuid-2", "10.0.0.2")}) @@ -60,14 +122,30 @@ func TestSameNameDistinctUUIDPeers_SurviveReplace(t *testing.T) { // TestSelfFilteredByUUID confirms the self-filter drops our own record by // hostUuid, keeping a same-named peer that holds a different UUID. func TestPeerDirectory_ExcludesLocalUUID(t *testing.T) { - src := newRelayPeerSource("self-uuid") + const selfUUID = "self-uuid" + const peerUUID = "peer-uuid" + const name = "host" + src := newRelayPeerSource(selfUUID) svc := map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceWorkload: {Port: 14320}} src.set([]noderec.DirectoryNode{ - {HostUUID: "self-uuid", Name: "host", IP: "10.0.0.1", Services: svc}, // us - {HostUUID: "peer-uuid", Name: "host", IP: "10.0.0.2", Services: svc}, // same name, different machine + { // us + IdentityVerification: noderec.IdentityLocal, + HostUUID: selfUUID, + Name: name, + IP: "10.0.0.1", + Services: svc, + }, + { // same name, different machine + IdentityVerification: noderec.IdentityPinned, + Trusted: true, + HostUUID: peerUUID, + Name: name, + IP: "10.0.0.2", + Services: svc, + }, }) nodes, err := src.Nodes(context.Background()) require.NoError(t, err) require.Len(t, nodes, 1) - assert.Equal(t, "peer-uuid", nodes[0].ID, "self-filter should drop only our own UUID") + assert.Equal(t, peerUUID, nodes[0].ID, "self-filter should drop only our own UUID") } diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index cb80450fd..76d1611cd 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -362,6 +362,7 @@ const ( NotifyNodeUpdated = "discovery:node-updated" NotifyNodeRemoved = "discovery:node-removed" NotifyNodeTelemetry = "discovery:node-telemetry" + NotifyNodeInfo = "discovery:node-info" // NotifyNodes is the broker relay -> subscriber push: the subscriber's full // filtered node set, re-sent on every change. Consumers replace their set @@ -484,6 +485,16 @@ type LocalNodeInfoParams struct { Info NodeInfo `json:"info"` } +// NodeInfoEvent carries hardware from an owned worker or authenticated peer. +// Record identity is assigned by the scanner, never taken from public HTTP. +type NodeInfoEvent struct { + DiscoveryRecordID string `json:"discoveryRecordId"` + HostUUID string `json:"hostUuid"` + ClusterUUID string `json:"clusterUuid,omitempty"` + IdentityVerification IdentityVerification `json:"identityVerification"` + Info NodeInfo `json:"info"` +} + // PeerMetadata comes from this host's local scanner registry, not mDNS input. type PeerMetadata struct { Name string `json:"name"` diff --git a/services/tests/broker_management_test.go b/services/tests/broker_management_test.go index 43e49fe69..5f737818b 100644 --- a/services/tests/broker_management_test.go +++ b/services/tests/broker_management_test.go @@ -19,6 +19,7 @@ import ( "io" "net" "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -28,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/appdir" + "nvpair-shared/noderec" ) // fakeOllama stands up a minimal Ollama-looking HTTP server on the fixed @@ -89,25 +91,25 @@ func persistLMStudioProxyPort(t *testing.T, port int) string { return root } -// fakeNodeInfo stands up a minimal node-info server on the fixed port the -// manual prober hits (14318): GET /v1/node-info returns 200 with the given -// hostUuid, so a manual node pointed at it learns its stable UUID. Skips the -// test if the port can't be bound. -func fakeNodeInfo(t *testing.T, hostUUID string) func() { +// fakePinnedNodeInfo serves the manual peer's UUID over mutually pinned TLS. +func fakePinnedNodeInfo(t *testing.T, cluster *interNodeCluster) int { t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:14318") - if err != nil { - t.Skipf("cannot bind fake node-info on 127.0.0.1:14318 (%v); skipping", err) - } mux := http.NewServeMux() mux.HandleFunc("/v1/node-info", func(w http.ResponseWriter, r *http.Request) { + if _, ok := cluster.peer.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", http.StatusForbidden) + return + } w.Header().Set("Content-Type", "application/json") - _, writeErr := io.WriteString(w, fmt.Sprintf(`{"GPUs":[],"hostUuid":%q}`, hostUUID)) - assert.NoError(t, writeErr) + if err := json.NewEncoder(w).Encode(noderec.NodeInfo{HostUUID: cluster.peerUUID, ClusterUUID: &cluster.peerUUID}); err != nil { + t.Errorf("encode peer node-info: %v", err) + } }) - srv := &http.Server{Handler: mux} - go func() { _ = srv.Serve(ln) }() - return func() { _ = srv.Close() } + srv := httptest.NewUnstartedServer(mux) + srv.TLS = cluster.peer.ServerTLSConfig() + srv.StartTLS() + t.Cleanup(srv.Close) + return srv.Listener.Addr().(*net.TCPAddr).Port } // proxyNode mirrors the proxy's nodes/list element (ollama-proxy Node). @@ -172,7 +174,7 @@ func TestBroker_BridgesManualNodeIntoProxy(t *testing.T) { select { case msg, ok := <-msgs: require.True(t, ok, "broker stream closed before the manual node bridged into the proxy") - if msg.Method == "" && msg.ID != nil && proxyNodesHas(t, msg.Result, nodeName) { + if msg.Method == "" && msg.ID != nil && proxyNodesHas(t, msg.Result, "manual:"+nodeName) { t.Logf("manual node %q bridged into proxy nodes/list", nodeName) return } @@ -185,8 +187,8 @@ func TestBroker_BridgesManualNodeIntoProxy(t *testing.T) { } } -// TestBrokerBridgesManualNodeUnderLearnedUUID: once a -// manual node's node-info reports its stable hostUuid, the proxy candidate must +// Once authenticated node-info reports a manual node's stable hostUuid, the +// proxy candidate must // be keyed by that UUID — the same operational key the discovery store and // scheduler use — so scheduler priority (node/set-priority) and scheduledOn // resolve to it. The candidate must therefore appear in ollama-proxy:nodes/list under @@ -197,10 +199,10 @@ func TestBroker_BridgesManualNodeUnderLearnedUUID(t *testing.T) { } stopOllama := fakeOllama(t) // skips if 11434 unavailable t.Cleanup(stopOllama) - stopNodeInfo := fakeNodeInfo(t, "learned-host-uuid") // skips if 14318 unavailable - t.Cleanup(stopNodeInfo) + cluster := newInterNodeCluster(t) + nodeInfoPort := fakePinnedNodeInfo(t, cluster) - stdin, msgs, _, cleanup := startBrokerWith(t, + stdin, msgs, _, cleanup := startBrokerWithDirs(t, t.TempDir(), cluster.nodeDir, "--manual-nodes-path", manualNodesBin, "--proxy-path", proxyBin, "--proxy-engines", "ollama", ) @@ -209,7 +211,7 @@ func TestBroker_BridgesManualNodeUnderLearnedUUID(t *testing.T) { waitForMethod(t, msgs, "app:ready", 10*time.Second) const nodeName = "xproc-manual-learned-uuid" - addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":940,"method":"node/add","params":{"address":"127.0.0.1","name":%q}}`, nodeName) + "\n" + addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":940,"method":"node/add","params":{"address":"127.0.0.1","name":%q,"tls_port":%d}}`, nodeName, nodeInfoPort) + "\n" _, err := stdin.Write([]byte(addReq)) require.NoError(t, err, "write node/add") @@ -223,7 +225,7 @@ func TestBroker_BridgesManualNodeUnderLearnedUUID(t *testing.T) { case msg, ok := <-msgs: require.True(t, ok, "broker stream closed before the manual node bridged under its learned UUID") if msg.Method == "" && msg.ID != nil { - if proxyNodesHas(t, msg.Result, "learned-host-uuid") { + if proxyNodesHas(t, msg.Result, cluster.peerUUID) { t.Logf("manual node bridged into proxy under learned hostUuid") return } @@ -270,7 +272,7 @@ func TestBroker_BridgesManualNodeIntoLMStudioProxy(t *testing.T) { select { case msg, ok := <-msgs: require.True(t, ok, "broker stream closed before the manual node bridged into lmstudio-proxy") - if msg.Method == "" && msg.ID != nil && proxyNodesHas(t, msg.Result, nodeName) { + if msg.Method == "" && msg.ID != nil && proxyNodesHas(t, msg.Result, "manual:"+nodeName) { t.Logf("manual node %q bridged into lmstudio-proxy nodes/list", nodeName) return } diff --git a/services/tests/cluster_data_plane_test.go b/services/tests/cluster_data_plane_test.go index 2e4380282..bdf30b203 100644 --- a/services/tests/cluster_data_plane_test.go +++ b/services/tests/cluster_data_plane_test.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -33,6 +34,7 @@ import ( "nvpair-shared/clustertrust" "nvpair-shared/jsonrpc" "nvpair-shared/netpick" + "nvpair-shared/noderec" "github.com/grandcat/zeroconf" ) @@ -80,18 +82,45 @@ func (c *interNodeCluster) clientAsPeer(t *testing.T) *http.Client { } // startStubClusterPeer runs a stub inter-node peer for this cluster over cluster -// mTLS and advertises it as a single _nvpair-node record carrying wl= and -// its cluster principal. The principal is what lets the node under test resolve -// its pin for the stub: the scanner annotates the browsed record as trusted and -// the workload-manager keys its pinned client on it. Without mTLS and that TXT -// key, the stub is simply not a routable cluster target and is never dialed. +// mTLS and advertises node-info and workload services on one port. Scanner must +// authenticate its inventory before workload-manager may use it as a peer, so +// the host UUID, cluster principal, and certificate identity must agree. // -// It returns the channel of frames the stub received. instance/hostUUID keep the -// record distinct from the node under test's own. -func (c *interNodeCluster) startStubClusterPeer(t *testing.T, instance, hostUUID string) <-chan jsonrpc.Message { +// It returns received frames and an idempotent stop function for node-loss +// tests. Cleanup also stops the peer. Its name and pinned UUID keep its record +// distinct from the node under test's own. +func (c *interNodeCluster) startStubClusterPeer(t *testing.T, instance string) (<-chan jsonrpc.Message, func()) { t.Helper() + addresses := netpick.LocalCandidates(netpick.Evidence{}) + require.NotEmpty(t, addresses, "stub peer needs a non-loopback address for discovery") + if len(addresses) > noderec.MaxAdvertisedIPs { + addresses = addresses[:noderec.MaxAdvertisedIPs] + } + // Listen on all interfaces: scanner and workload-manager use the advertised + // local interface addresses rather than the test's loopback address. + ln, err := net.Listen("tcp", ":0") + require.NoError(t, err, "stub peer listen") + port := ln.Addr().(*net.TCPAddr).Port + received := make(chan jsonrpc.Message, 128) mux := http.NewServeMux() + mux.HandleFunc("/v1/node-info", func(w http.ResponseWriter, r *http.Request) { + if _, ok := c.peer.VerifyClientPin(r); !ok { + http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) + return + } + info := noderec.NodeInfo{ + HostUUID: c.peerUUID, ClusterUUID: &c.peerUUID, + Peer: &noderec.PeerMetadata{ + Name: instance, Addresses: addresses, + Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port, noderec.ServiceWorkload: port}, + }, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(info); err != nil { + t.Errorf("encode stub peer inventory: %v", err) + } + }) mux.HandleFunc("/v1/workloads/events", func(w http.ResponseWriter, r *http.Request) { // The sender must present the leaf this peer pins — the outbound half of // the gate, so a broadcast is authenticated in both directions. @@ -111,22 +140,27 @@ func (c *interNodeCluster) startStubClusterPeer(t *testing.T, instance, hostUUID w.WriteHeader(http.StatusOK) }) - // Listen on all interfaces: zeroconf advertises the host's real interface - // IP(s), so the manager dials the peer there, not at 127.0.0.1. - ln, err := net.Listen("tcp", ":0") - require.NoError(t, err, "stub peer listen") - port := ln.Addr().(*net.TCPAddr).Port srv := &http.Server{Handler: mux} go srv.Serve(tls.NewListener(ln, c.peer.ServerTLSConfig())) t.Cleanup(func() { _ = srv.Close() }) - txt := []string{"v=1", "uuid=" + hostUUID, fmt.Sprintf("wl=%d", port), + txt := []string{"v=1", "uuid=" + c.peerUUID, "ip=" + addresses[0], "ips=" + strings.Join(addresses, ","), + fmt.Sprintf("ni=%d", port), fmt.Sprintf("wl=%d", port), clustertrust.ClusterUUIDTXTKey + "=" + c.peerUUID} adv, err := zeroconf.Register(instance, nodeRecordService, testDomain, port, txt, nil) require.NoError(t, err, "register stub peer") - t.Cleanup(adv.Shutdown) - t.Logf("stub cluster peer advertising %s (wl=%d, cluster-uuid=%s)", nodeRecordService, port, c.peerUUID) - return received + var stopped sync.Once + stop := func() { + stopped.Do(func() { + adv.Shutdown() + if err := srv.Close(); err != nil { + t.Errorf("close stub peer: %v", err) + } + }) + } + t.Cleanup(stop) + t.Logf("stub cluster peer advertising %s (ni=%d, wl=%d, cluster-uuid=%s)", nodeRecordService, port, port, c.peerUUID) + return received, stop } // waitTCP blocks until addr accepts a TCP connection. Every "this request must be diff --git a/services/tests/remote_engine_test.go b/services/tests/remote_engine_test.go index 618919c16..edd2a309a 100644 --- a/services/tests/remote_engine_test.go +++ b/services/tests/remote_engine_test.go @@ -116,12 +116,13 @@ func TestRemoteEngine_ReturnsInstalledEnginesForPeer(t *testing.T) { // synchronously before the next frame, so the directory is populated by the // time the remote request is read. snapshot := fmt.Sprintf(`{"jsonrpc":"2.0","method":"discovery:nodes","params":{"nodes":[`+ - `{"hostUuid":"nodeB","name":"nodeB","ip":"127.0.0.1","clusterUuid":%q,"trusted":true,`+ - `"services":{"ec":{"port":%d}},"lastSeen":0}]}}`, uuidB, portB) + `{"hostUuid":%q,"name":"nodeB","ip":"127.0.0.1","clusterUuid":%q,"trusted":true,`+ + `"identityVerification":"pinned","discoveryRecordId":"peer-b",`+ + `"services":{"ec":{"port":%d}},"lastSeen":0}]}}`, uuidB, uuidB, portB) writeRawFrame(t, aStdin, snapshot) // Fire the remote read and await the response. - writeRawFrame(t, aStdin, `{"jsonrpc":"2.0","id":1,"method":"engine:remote-get-installed","params":{"node":"nodeB"}}`) + writeRawFrame(t, aStdin, fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"engine:remote-get-installed","params":{"node":%q}}`, uuidB)) resp := waitForResponse(t, aMsgs, 15*time.Second) require.Nil(t, resp.Error, "remote-get-installed errored") var res struct { @@ -156,13 +157,15 @@ func TestRemoteEngine_RejectsUntrusted(t *testing.T) { waitForMethod(t, aMsgs, "engine:ready", 10*time.Second) snapshot := fmt.Sprintf(`{"jsonrpc":"2.0","method":"discovery:nodes","params":{"nodes":[`+ - `{"hostUuid":"nodeB","name":"nodeB","ip":"127.0.0.1","clusterUuid":%q,"trusted":true,`+ - `"services":{"ec":{"port":%d}},"lastSeen":0}]}}`, uuidB, portB) + `{"hostUuid":%q,"name":"nodeB","ip":"127.0.0.1","clusterUuid":%q,"trusted":true,`+ + `"identityVerification":"pinned","discoveryRecordId":"peer-b",`+ + `"services":{"ec":{"port":%d}},"lastSeen":0}]}}`, uuidB, uuidB, portB) writeRawFrame(t, aStdin, snapshot) - writeRawFrame(t, aStdin, `{"jsonrpc":"2.0","id":1,"method":"engine:remote-get-installed","params":{"node":"nodeB"}}`) + writeRawFrame(t, aStdin, fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"engine:remote-get-installed","params":{"node":%q}}`, uuidB)) resp := waitForResponse(t, aMsgs, 15*time.Second) require.NotNil(t, resp.Error, "expected an error from an unpinned caller, got result") + require.Contains(t, resp.Error.Message, "HTTP 403", "the discovered peer must reject the caller at its pin gate") t.Logf("unpinned caller correctly refused: %+v", resp.Error) } diff --git a/services/tests/secure_inference_test.go b/services/tests/secure_inference_test.go index 0fd77fbba..fc87fad30 100644 --- a/services/tests/secure_inference_test.go +++ b/services/tests/secure_inference_test.go @@ -176,32 +176,34 @@ func (p *proxyProc) setLocalBackend(engine, host string, port int, healthy bool) func (p *proxyProc) pushOllamaPeer(name, ip, clusterUUID string, proxyPort int, models []string) { p.notify(engines.AddressMethod("ollama", "discovery:nodes"), map[string]any{ "nodes": []map[string]any{{ - "hostUuid": name, - "name": name, - "ip": ip, - "clusterUuid": clusterUUID, - "trusted": true, - "services": map[string]any{"ol": map[string]any{"port": proxyPort}}, - "modelsByEngine": map[string]any{"ollama": models}, - "lastSeen": time.Now().Unix(), + "hostUuid": clusterUUID, + "discoveryRecordId": "peer:" + clusterUUID, + "identityVerification": "pinned", + "name": name, + "ip": ip, + "clusterUuid": clusterUUID, + "trusted": true, + "services": map[string]any{"ol": map[string]any{"port": proxyPort}}, + "modelsByEngine": map[string]any{"ollama": models}, + "lastSeen": time.Now().Unix(), }}, }) } -// waitForRoutableNode polls the proxy's nodes/list until it reports at least one -// routable node, so the async discovery:nodes push has been applied before the -// test issues a request. -func (p *proxyProc) waitForRoutableNode(t *testing.T) { +// waitForRoutableNode waits for the intended peer before issuing inference. +func (p *proxyProc) waitForRoutableNode(t *testing.T, nodeUUID string) { t.Helper() deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { resp := p.call(engines.AddressMethod("ollama", "nodes/list"), nil) var r struct { - Nodes []json.RawMessage `json:"nodes"` + Nodes []proxyNode `json:"nodes"` } require.NoError(t, json.Unmarshal(resp.Result, &r)) - if len(r.Nodes) > 0 { - return + for _, node := range r.Nodes { + if node.ID == nodeUUID { + return + } } time.Sleep(100 * time.Millisecond) } @@ -254,8 +256,7 @@ func TestClusterInference_RequiresPinnedMutualTLS(t *testing.T) { aInfo := decodeResult[cmNodeID](t, cmA.call("cluster:get-node-id", nil)) bInfo := decodeResult[cmNodeID](t, cmB.call("cluster:get-node-id", nil)) - // The cluster-manager listeners need a moment to bind before pairing. - time.Sleep(time.Second) + waitForPort(t, "127.0.0.1", cmBPort, 10*time.Second) // A founds a cluster and pairs with B (mutual pins land on disk). C founds // its own cluster and never pairs with A or B — it is a foreign node. @@ -275,7 +276,7 @@ func TestClusterInference_RequiresPinnedMutualTLS(t *testing.T) { defer proxyA.stop() // Tell A that B runs ollama at B's promoted proxy port, as a trusted member. proxyA.pushOllamaPeer("node-b", "127.0.0.1", bInfo.NodeUUID, proxyB.port, []string{"m:latest"}) - proxyA.waitForRoutableNode(t) + proxyA.waitForRoutableNode(t, bInfo.NodeUUID) genBody := []byte(`{"model":"m:latest","prompt":"hi","stream":false}`) diff --git a/services/tests/workload_identity_interop_test.go b/services/tests/workload_identity_interop_test.go index b3a0d7472..34ea77096 100644 --- a/services/tests/workload_identity_interop_test.go +++ b/services/tests/workload_identity_interop_test.go @@ -202,7 +202,7 @@ func TestWorkloadManager_RehydratesActiveWorkloadOnRestart(t *testing.T) { // manager fans out to it. The inter-node interface is cluster mTLS only, so the // stub must hold a cluster identity the node under test pins. fx := newInterNodeCluster(t) - received := fx.startStubClusterPeer(t, "rehydrate-wm-peer", "rehydrate-peer-uuid") + received, _ := fx.startStubClusterPeer(t, "rehydrate-wm-peer") stdin, msgs, stderr, cleanup := startBrokerWithDirs(t, t.TempDir(), fx.nodeDir, "--proxy-path", proxyBin, "--proxy-engines", "ollama", @@ -304,7 +304,7 @@ func TestWorkloadManager_RehydratesRecentTerminalOnRestart(t *testing.T) { ollamaPort := portOfURL(t, ollama.URL) fx := newInterNodeCluster(t) - received := fx.startStubClusterPeer(t, "rehydrate-term-peer", "rehydrate-term-peer-uuid") + received, _ := fx.startStubClusterPeer(t, "rehydrate-term-peer") stdin, msgs, stderr, cleanup := startBrokerWithDirs(t, t.TempDir(), fx.nodeDir, "--proxy-path", proxyBin, "--proxy-engines", "ollama", diff --git a/services/tests/workload_interop_test.go b/services/tests/workload_interop_test.go index 6a687b921..6e6a24aa7 100644 --- a/services/tests/workload_interop_test.go +++ b/services/tests/workload_interop_test.go @@ -40,8 +40,7 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/jsonrpc" - - "github.com/grandcat/zeroconf" + "nvpair-shared/noderec" ) const workloadMgrService = "_nvpair-workload-manager._tcp" @@ -239,9 +238,10 @@ func TestWorkloadRelay_FailsActiveWorkWhenNodeDisappears(t *testing.T) { // must match on that UUID, not the display name. Keying by name would leave // a UUID-stamped workload running after the (name-distinct) node departs. const peerNode = "nodeloss-peer" - const peerUUID = "nodeloss-peer-uuid" fx := newInterNodeCluster(t) + peerUUID := fx.peerUUID + _, stopPeer := fx.startStubClusterPeer(t, peerNode) stdin, msgs, cleanup := startBrokerProcInCluster(t, fx.nodeDir, "--scanner-path", scannerBin, "--workload-manager-path", workloadMgrBin, @@ -259,26 +259,10 @@ func TestWorkloadRelay_FailsActiveWorkWhenNodeDisappears(t *testing.T) { require.True(t, sr.Subscribed, "workloads:subscribe ack") t.Log("subscribed to workloads stream") - // Advertise the peer as an identity-only _nvpair-node record (no service - // ports), so once its advertisement is withdrawn the scanner's liveness - // probe has nothing to dial and evicts it cleanly. A distinct uuid keeps - // it from colliding with this host's own record. - peerTXT := []string{"v=1", "uuid=" + peerUUID, "ip=127.0.0.1"} - peerAdv, err := zeroconf.Register(peerNode, nodeRecordService, testDomain, 14999, peerTXT, nil) - require.NoError(t, err, "register peer node") - stopAdv := func() { - if peerAdv != nil { - peerAdv.Shutdown() - peerAdv = nil - } - } - t.Cleanup(stopAdv) - t.Logf("advertising peer %q as %s", peerNode, nodeRecordService) - - // The peer must be in the broker's directory before its removal can - // trigger a sweep, so wait until discovery has folded it in. - pollNodeListed(t, stdin, msgs, peerNode, 1000, 30*time.Second, nil) - t.Logf("peer %q discovered", peerNode) + // Only an authenticated peer's removal may fail its workloads. An + // unverified advertisement cannot supply evidence that a known host left. + waitForPinnedPeer(t, stdin, msgs, peerUUID, 30*time.Second) + t.Logf("peer %q authenticated", peerNode) // A peer-origin running workload (scheduled on that same peer) POSTed to // the manager's inter-node endpoint. The broker relays it to us as a @@ -298,8 +282,8 @@ func TestWorkloadRelay_FailsActiveWorkWhenNodeDisappears(t *testing.T) { // The window is deliberately long enough to outlast a node saturated by its // own inference load, so this deadline has to clear it with room for the // probes and scan jitter. - stopAdv() - t.Log("peer advertisement withdrawn; awaiting node eviction + workload failure") + stopPeer() + t.Log("peer advertisement and listener stopped; awaiting node eviction + workload failure") failed := waitForWorkloadEvent(t, msgs, "workloads:upsert", "wl-nodeloss-1", 2*time.Minute) require.Equal(t, "failed", failed.WorkloadInfo.State, "post-eviction upsert state") @@ -336,7 +320,7 @@ func TestWorkloadManager_BroadcastsLocalEventsToPeers(t *testing.T) { // _nvpair-node record and pushes it to the subscribed workload-manager's relay // peer source, which is the path that replaced the per-service browse. fx := newInterNodeCluster(t) - received := fx.startStubClusterPeer(t, "stub-wm-peer", "stub-wm-peer-uuid") + received, _ := fx.startStubClusterPeer(t, "stub-wm-peer") stdin, msgs, cleanup := startBrokerProcInCluster(t, fx.nodeDir, "--scanner-path", scannerBin, @@ -360,9 +344,12 @@ func TestWorkloadManager_BroadcastsLocalEventsToPeers(t *testing.T) { waitForResponse(t, msgs, 5*time.Second) t.Log("fake ollama pinned as proxy target") + waitForPinnedPeer(t, stdin, msgs, fx.peerUUID, 30*time.Second) + t.Log("stub peer authenticated in discovery") + // Fire inference requests until the stub peer records a broadcast. The - // retry loop absorbs mDNS discovery latency (the manager refreshes its - // peer set every 5 s and only broadcasts to peers it already knows). + // retry loop covers workload-manager's five-second target refresh after + // authenticated discovery has supplied the peer. client := &http.Client{Timeout: 3 * time.Second} fire := func() { resp, err := client.Post( @@ -370,11 +357,12 @@ func TestWorkloadManager_BroadcastsLocalEventsToPeers(t *testing.T) { "application/json", strings.NewReader(`{"model":"test-model","messages":[]}`), ) - if err == nil { - _, copyErr := io.Copy(io.Discard, resp.Body) - require.NoError(t, copyErr) - resp.Body.Close() - } + require.NoError(t, err, "submit inference through the ready proxy") + _, readErr := io.Copy(io.Discard, resp.Body) + closeErr := resp.Body.Close() + require.NoError(t, readErr, "read inference response") + require.NoError(t, closeErr, "close inference response") + require.Equal(t, http.StatusOK, resp.StatusCode, "inference response status") } fire() @@ -405,6 +393,48 @@ func TestWorkloadManager_BroadcastsLocalEventsToPeers(t *testing.T) { } } +// waitForPinnedPeer distinguishes an authenticated peer from its unverified +// mDNS candidate before a test submits work that must reach that peer. +func waitForPinnedPeer(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, hostUUID string, timeout time.Duration) { + t.Helper() + timer := time.NewTimer(timeout) + defer timer.Stop() + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + id := 1000 + sendReq(t, stdin, id, "discovery:get-nodes") + for { + select { + case msg, ok := <-msgs: + require.True(t, ok, "broker stream closed before peer authentication") + if msg.Method != "" || msg.ID == nil { + continue + } + require.Nil(t, msg.Error, "discovery:get-nodes") + var result struct { + Nodes []struct { + HostUUID string `json:"hostUuid"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + Trusted bool `json:"trusted"` + } `json:"nodes"` + } + require.NoError(t, json.Unmarshal(msg.Result, &result), "discovery:get-nodes") + for _, node := range result.Nodes { + if node.HostUUID == hostUUID && node.IdentityVerification == noderec.IdentityPinned && node.Trusted { + require.NotEmpty(t, node.DiscoveryRecordID) + return + } + } + case <-ticker.C: + id++ + sendReq(t, stdin, id, "discovery:get-nodes") + case <-timer.C: + require.FailNowf(t, "peer did not authenticate in discovery", "hostUuid=%s", hostUUID) + } + } +} + // postPeerEvent POSTs a peer workload frame to the manager's inter-node endpoint // as a pinned cluster peer, retrying until it returns 200 (the listener comes up // in a goroutine after the broker spawns the manager) or the timeout elapses. The From de51667698f87bec2d6195dd70765c025ff057ea Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Wed, 21 Oct 2026 16:07:54 -0400 Subject: [PATCH 6/8] Invalidate peer proofs across trust changes and worker restarts Withdraw verified scanner and manual records, hardware snapshots, and scheduling samples when their pin or local admission changes. Bind opaque discovery record IDs to the current certificates and admissions so queued results and delayed removals cannot revive or delete the wrong proof. Retain the last known admission on filesystem read failures while honoring absent, cleared, or malformed admission records. Clear scanner state on crash and reject retired manual/node-info worker callbacks. Preserve owned local sample age, manual status age, and elapsed time during scheduler replay. Keep explicitly configured manual engine targets independent, and prevent late proxy presence from recreating a withdrawn desktop peer. Retire workloads when peer authorization is withdrawn, regardless of whether a manual update, worker cleanup, or the trust notification arrives first. Preserve jobs when trust is unchanged or another source has a current authenticated record. Discard a manual record's previous measurements before publishing its replacement so scheduler replay cannot reuse them under the new authorization. Cached liveness also checks the current admission-bound record ID before using inventory or inference activity. TestPeerReachable_RejectsCachedLivenessAfterReadmission covers same-certificate readmission before reload notification and recovery after reauthentication. Suggested review order: 1. Scanner and manual-nodes specifications as reference contracts, then scanner, manual-nodes, and broker READMEs: lifecycle behavior. 2. services/shared/clustertrust/clustertrust_test.go and mesh_test.go, then membership.go, clustertrust.go, and mesh.go: independent certificate bytes, certificate/admission bindings, serialized reloads, and preservation of authorization during admission read failures. 3. services/nvpair-node-scanner/daemon_test.go, then daemon.go: exact-certificate rejection, invalidated inventory/model results, reauthentication, current-record liveness, and record-specific cache cleanup. 4. services/nvpair-manual-nodes/manager_test.go, then manager.go: in-flight identity/entry changes, current pin binding, and age-preserving cached statuses. 5. services/nvpair-ui-broker/broker_test.go trust, workload, and restart cases, manualnodes_test.go, and nodeinfo_observed_test.go, then broker.go, discovery.go, and manualnodes.go: source withdrawal, workload cleanup across notification orders, preservation of current peers, and retired worker callbacks. 6. services/nvpair-ui-broker/broker_test.go manual record replacement case, discovery_test.go, nodeinfo_test.go, and telemetry_test.go, then the corresponding broker.go paths, discovery.go, nodeinfo.go, and telemetry.go: replacement measurements, scheduler baselines, and age-preserving replay. 7. services/nvpair-ui-broker/proxyaddressing_test.go, then proxy.go and engineproxy.go: blocked proxy writes cannot stall discovery or trust withdrawal, and each engine retains its latest authoritative subscription snapshot. 8. services/nvpair-node-info/main_test.go: real private IPC and public display across join and leave, including admission clearing while a reader holds the file open on Windows. 9. desktop/tests/modular/node-identity-contract.test.ts, then desktop/src/electron/service-bridge/modular-state.ts: immediate row withdrawal, late proxy suppression, and restoration by a new broker record. Review validation: observed failing regressions before the admission-read, replay, workload-cleanup, notification-order, and Windows fixture fixes. Full shared, broker, and node-info Go suites and focused scanner/manual authentication regressions pass. Formatting and whitespace checks pass. Original broader validation, before the review fixes: shared and eleven affected service Go suites; five controlled services/tests cases and real node-info child coverage; desktop tests with 48 files, 281 passed and two platform skips, plus 26 VC runtime checks; typecheck, lint, dead-code, contracts, all 13 Windows x64 service builds, formatting, whitespace, and SPDX checks. The unchanged engine-manager Windows chmod fixture is excluded from its broad run. Race instrumentation is unavailable because cgo is disabled. The user accepts best-effort freshness across raw worker-stdio buffering; no timestamp contract change is planned. Compact telemetry remains unchanged. Signed-off-by: Kaylee Lubick --- .../electron/service-bridge/modular-state.ts | 14 + .../modular/node-identity-contract.test.ts | 36 +- services/nvpair-manual-nodes/README.md | 9 +- services/nvpair-manual-nodes/manager.go | 140 ++++- services/nvpair-manual-nodes/manager_test.go | 145 ++++- services/nvpair-node-info/main_test.go | 40 ++ services/nvpair-node-scanner/README.md | 2 + services/nvpair-node-scanner/daemon.go | 41 +- services/nvpair-node-scanner/daemon_test.go | 57 +- services/nvpair-ui-broker/README.md | 11 + services/nvpair-ui-broker/broker.go | 249 +++++++- services/nvpair-ui-broker/broker_test.go | 549 ++++++++++++++++++ services/nvpair-ui-broker/discovery.go | 91 ++- services/nvpair-ui-broker/discovery_test.go | 85 ++- services/nvpair-ui-broker/engineproxy.go | 2 + services/nvpair-ui-broker/manualnodes.go | 179 +++++- services/nvpair-ui-broker/manualnodes_test.go | 322 ++++++++++ services/nvpair-ui-broker/nodeinfo.go | 3 +- .../nodeinfo_observed_test.go | 2 +- services/nvpair-ui-broker/nodeinfo_test.go | 57 +- services/nvpair-ui-broker/proxy.go | 52 +- .../nvpair-ui-broker/proxyaddressing_test.go | 185 ++++++ services/nvpair-ui-broker/telemetry.go | 26 +- services/nvpair-ui-broker/telemetry_test.go | 60 ++ services/shared/clustertrust/clustertrust.go | 56 +- .../shared/clustertrust/clustertrust_test.go | 19 + services/shared/clustertrust/membership.go | 25 +- services/shared/clustertrust/mesh.go | 40 +- services/shared/clustertrust/mesh_test.go | 243 ++++++-- 29 files changed, 2536 insertions(+), 204 deletions(-) diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index c8be77aa0..a82e75294 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -890,6 +890,7 @@ function sameModelsByEngine( class ModularBridgeState { private nodes = new Map() private brokerNodeIds = new Set() + private withdrawnPeerIds = new Set() private errors: ServiceError[] = [] private workloads = new Map() private logs: LogEntry[] = [] @@ -2394,6 +2395,12 @@ class ModularBridgeState { const existing = this.nodes.get(nodeId) if (!existing) return + if (existing.identityVerification === 'pinned') { + this.withdrawnPeerIds.add(nodeId) + this.removeNodeEntry(nodeId) + return + } + const remainingSources = removeSource(existing.sources, 'broker') if (remainingSources.length === 0) { this.removeNodeEntry(nodeId) @@ -2427,6 +2434,13 @@ class ModularBridgeState { } private upsertNode(node: ModularNode, source?: BrokerNodeSource): void { + if (source === 'broker') { + this.withdrawnPeerIds.delete(node.id) + } else if (this.withdrawnPeerIds.has(node.id)) { + // Only a new broker record may restore a withdrawn peer. Proxy + // presence can arrive late from a different subprocess stream. + return + } // Self is identified authoritatively by the cluster-manager's node UUID // (`cluster:get-node-id`), which equals the discovery/proxy hostUuid key, // so there is no hostname guessing here — `setSelfId` is the sole source. diff --git a/desktop/tests/modular/node-identity-contract.test.ts b/desktop/tests/modular/node-identity-contract.test.ts index 4ff3c983e..5e924580d 100644 --- a/desktop/tests/modular/node-identity-contract.test.ts +++ b/desktop/tests/modular/node-identity-contract.test.ts @@ -202,8 +202,8 @@ describe('node identity contract', () => { ) }) // Let discovery and a proxy both report one host, then withdraw discovery. - // The proxy may keep the row visible, but cannot retain broker verification. - it('clears withdrawn broker provenance while proxy presence survives', () => { + // Cached proxy presence cannot retain a withdrawn verified peer row. + it('removes a withdrawn verified peer despite cached proxy presence', () => { const hostUuid = 'identity-withdrawn-host' const hostName = 'workstation' const address = '192.0.2.21' @@ -239,9 +239,35 @@ describe('node identity contract', () => { }) const node = state.getAvailableNodes().find(node => node.id === hostUuid) - expect(node).toBeDefined() - expect(node).not.toHaveProperty('discoveryRecordId') - expect(node).not.toHaveProperty('identityVerification') + expect(node).toBeUndefined() + + // A proxy can still have an older discovery notification queued when + // the broker withdraws the row. It cannot restore that former peer. + state.handleNotification({ + source: 'ollama-proxy', + method: 'node/discovered', + params: { id: hostUuid, host: hostName, port: 11434, addresses: [address] } + }) + expect(state.getAvailableNodes().find(node => node.id === hostUuid)).toBeUndefined() + + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: [ + { + hostUuid, + name: hostName, + ipAddress: address, + port: 14318, + discoveryRecordId: 'new-proof', + identityVerification: 'pinned' + } + ] + }) + expect(state.getAvailableNodes().find(node => node.id === hostUuid)).toMatchObject({ + discoveryRecordId: 'new-proof', + identityVerification: 'pinned' + }) }) // A proxy's host UUID and an unverified discovery record identify separate diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index e7f5fdc57..6874d7b50 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -53,7 +53,8 @@ Emitted when a manually added node has been probed and its initial status determ "telemetryValid":true, "msSince":120, "hostUuid":"stable-node-uuid", - "identityVerification":"pinned" + "identityVerification":"pinned", + "discoveryRecordId":"opaque-proof-bound-record-id" } } ``` @@ -66,6 +67,12 @@ Emitted when a periodic probe detects a change, and for every authenticated hardware sample even when values are unchanged. `telemetryValid` and `msSince` preserve node-info's distinction between an idle 0% sample and missing telemetry so the broker can feed manual nodes into the same scheduler cache. +Cached status replies and initial discovery notifications add elapsed time to +the original sample age. Pinned statuses carry an opaque `discoveryRecordId` +bound to the local identity/admission and peer pin/admission. Pin changes, +departure, or identity replacement withdraw that proof and all hardware fields; +configured engine endpoints remain independently available. In-flight probes +cannot update an entry removed and re-added under the same ID. ### `node/removed` diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 9b326885a..b0740f5ff 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -5,6 +5,7 @@ package main import ( "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -12,6 +13,7 @@ import ( "log/slog" "net" "net/http" + "net/http/httptrace" "strconv" "sync" "time" @@ -68,11 +70,13 @@ type MemoryInfo struct { } type NodeInfoResponse struct { - GPUs []GPUInfo `json:"GPUs"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` + discoveryRecordID string + receivedAt time.Time + GPUs []GPUInfo `json:"GPUs"` + CPU *CPUInfo `json:"cpu,omitempty"` + Memory *MemoryInfo `json:"memory,omitempty"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` // HostUUID is the remote node's stable per-host identity, reported by // node-info. We surface it so a manual node is tracked by the same permanent // identity as the rest of the fleet — including deduping with the same @@ -100,6 +104,7 @@ type ManualEntry struct { // they do not describe whether the hardware probe uses TLS (it always does). // NodeInfoPort is the attempted HTTPS port, even when that probe fails. type ManualNodeStatus struct { + DiscoveryRecordID string `json:"discoveryRecordId,omitempty"` IdentityVerification noderec.IdentityVerification `json:"identityVerification"` ID string `json:"id"` Name string `json:"name,omitempty"` @@ -131,7 +136,8 @@ type ManualNodeStatus struct { // sameAvailability compares identity, configuration, engine models, and service // availability. Every authenticated hardware sample is published independently. func (s ManualNodeStatus) sameAvailability(other ManualNodeStatus) bool { - return s.IdentityVerification == other.IdentityVerification && + return s.DiscoveryRecordID == other.DiscoveryRecordID && + s.IdentityVerification == other.IdentityVerification && s.ID == other.ID && s.Name == other.Name && s.Address == other.Address && @@ -148,6 +154,15 @@ func (s ManualNodeStatus) sameAvailability(other ManualNodeStatus) bool { s.HostUUID == other.HostUUID } +func (s ManualNodeStatus) withoutVerification() ManualNodeStatus { + s.IdentityVerification = noderec.IdentityUnverified + s.DiscoveryRecordID, s.HostUUID = "", "" + s.NodeInfoUp = false + s.GPUs, s.CPU, s.Memory = nil, nil, nil + s.TelemetryValid, s.MSSince = false, 0 + return s +} + type ReadyParams struct { Version string `json:"version"` } @@ -163,6 +178,16 @@ type trackedNode struct { // probeFailThreshold so a single transient failure doesn't // generate UI noise. consecutiveFails int + observedAt time.Time +} + +// statusAt projects cached hardware at send time without resetting its age. +func (n *trackedNode) statusAt(now time.Time) ManualNodeStatus { + status := n.status + if status.TelemetryValid { + status.MSSince = (noderec.NodeInfo{TelemetryValid: true, MSSince: status.MSSince}).Aged(now.Sub(n.observedAt)).MSSince + } + return status } // Manager keeps manual entries in memory and publishes their probe results. @@ -222,10 +247,42 @@ func (m *Manager) Run(ctx context.Context) error { } go m.probeLoop(ctx) + go m.watchTrust(ctx) return m.readLoop(ctx) } +func (m *Manager) watchTrust(ctx context.Context) { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.reconcileTrust() + } + } +} + +// reconcileTrust withdraws cached proofs independently of slow engine probes. +// Status writes and notifications share mu with removal and probe completion. +func (m *Manager) reconcileTrust() { + m.mu.Lock() + defer m.mu.Unlock() + m.mesh.Refresh() + for _, node := range m.nodes { + status := node.status + if status.IdentityVerification == noderec.IdentityPinned && (status.DiscoveryRecordID == "" || status.DiscoveryRecordID != m.mesh.PeerRecordID(status.HostUUID)) { + node.status = status.withoutVerification() + if err := m.codec.Notify("node/updated", node.status); err != nil { + slog.Warn("failed to withdraw manual hardware", "node_id", status.ID, "err", err) + } + } + } + m.peerClients.DropUnpinned() +} + func (m *Manager) probeLoop(ctx context.Context) { ticker := time.NewTicker(probeInterval) defer ticker.Stop() @@ -264,6 +321,12 @@ func (m *Manager) probeAll(ctx context.Context) { func (m *Manager) probeNode(entry ManualEntry) { addr := entry.Address id := nodeID(entry) + m.mu.RLock() + tracked := m.nodes[id] + m.mu.RUnlock() + if tracked == nil || tracked.entry != entry { + return + } ollamaUp, ollamaModels := m.probeOllama(addr, 11434) lmStudioUp, lmStudioModels := m.probeLMStudio(addr, lmStudioPort) @@ -278,6 +341,7 @@ func (m *Manager) probeNode(entry ManualEntry) { // uses its host's exact pin; an unknown endpoint may present any current // pin, then must bind its response identities to that certificate. m.mesh.Refresh() + localRecordID := m.mesh.PeerRecordID(m.mesh.NodeUUID()) nodeInfoUp, info := false, NodeInfoResponse{} m.mu.RLock() var knownUUID string @@ -286,7 +350,7 @@ func (m *Manager) probeNode(entry ManualEntry) { } m.mu.RUnlock() if client, ok := m.peerClients.Client(knownUUID); knownUUID != "" && ok { - nodeInfoUp, info = m.probeNodeInfo(client, addr, nodeInfoPort) + nodeInfoUp, info = m.probeNodeInfo(client, addr, nodeInfoPort, localRecordID) } else if cfg, ok := m.mesh.ClientTLSConfigAny(); ok { client := &http.Client{ Timeout: probeTimeout, @@ -298,11 +362,12 @@ func (m *Manager) probeNode(entry ManualEntry) { return http.ErrUseLastResponse }, } - nodeInfoUp, info = m.probeNodeInfo(client, addr, nodeInfoPort) + nodeInfoUp, info = m.probeNodeInfo(client, addr, nodeInfoPort, localRecordID) } m.peerClients.DropUnpinned() newStatus := ManualNodeStatus{ + DiscoveryRecordID: info.discoveryRecordID, IdentityVerification: noderec.IdentityUnverified, ID: id, Name: entry.Name, @@ -324,31 +389,35 @@ func (m *Manager) probeNode(entry ManualEntry) { MSSince: info.MSSince, HostUUID: info.HostUUID, } - if nodeInfoUp { + if newStatus.NodeInfoUp { newStatus.IdentityVerification = noderec.IdentityPinned } - reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.NodeInfoUp - m.mu.Lock() tn, exists := m.nodes[id] - if !exists { + if !exists || tn != tracked { m.mu.Unlock() return } + defer m.mu.Unlock() + m.mesh.Refresh() + if newStatus.NodeInfoUp && (newStatus.DiscoveryRecordID == "" || newStatus.DiscoveryRecordID != m.mesh.PeerRecordID(newStatus.HostUUID)) { + newStatus = newStatus.withoutVerification() + } + reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.NodeInfoUp prev := tn.status prevFails := tn.consecutiveFails // A failed authenticated read clears service measurements and proof. The // configured engine target remains independently available when it answers. tn.status = newStatus + tn.observedAt = info.receivedAt if reachable { tn.consecutiveFails = 0 } else { tn.consecutiveFails++ } curFails := tn.consecutiveFails - m.mu.Unlock() changed := !prev.sameAvailability(newStatus) @@ -362,8 +431,8 @@ func (m *Manager) probeNode(entry ManualEntry) { "node_id", id, "addr", addr, "ollama_up", newStatus.OllamaUp, "node_info_up", newStatus.NodeInfoUp) } - if changed || nodeInfoUp { - if err := m.codec.Notify("node/updated", newStatus); err != nil { + if changed || newStatus.NodeInfoUp { + if err := m.codec.Notify("node/updated", tn.statusAt(time.Now())); err != nil { slog.Warn("failed to publish manual node status", "node_id", id, "err", err) } } @@ -516,10 +585,21 @@ func (m *Manager) fetchOllamaModels(addr string, port int) []string { // Success means the remote identity was authenticated, not that GPU telemetry // is valid. Any transport, decoding, or identity failure returns false and an // empty response so stale or unverified measurements cannot survive the probe. -func (m *Manager) probeNodeInfo(client *http.Client, addr string, port int) (bool, NodeInfoResponse) { +func (m *Manager) probeNodeInfo(client *http.Client, addr string, port int, localRecordID string) (bool, NodeInfoResponse) { url := "https://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/node-info" start := time.Now() - request, err := http.NewRequest(http.MethodGet, url, nil) + // Capture authorization before writing the request, including on reused + // connections and when the endpoint's peer UUID was initially unknown. + var recordID string + trace := &httptrace.ClientTrace{GotConn: func(connection httptrace.GotConnInfo) { + if conn, ok := connection.Conn.(*tls.Conn); ok { + state := conn.ConnectionState() + if len(state.PeerCertificates) > 0 { + recordID = m.mesh.PeerRecordID(clustertrust.UUIDFromCert(state.PeerCertificates[0])) + } + } + }} + request, err := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), http.MethodGet, url, nil) if err != nil { return false, NodeInfoResponse{} } @@ -554,6 +634,9 @@ func (m *Manager) probeNodeInfo(client *http.Client, addr string, port int) (boo return false, NodeInfoResponse{} } m.mesh.Refresh() + if localRecordID == "" || localRecordID != m.mesh.PeerRecordID(m.mesh.NodeUUID()) { + return false, NodeInfoResponse{} + } if resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { // Without the server's TLS certificate, the response has no identity // proof to check against our pins. @@ -574,11 +657,16 @@ func (m *Manager) probeNodeInfo(client *http.Client, addr string, port int) (boo // may have been removed or replaced since the TLS handshake completed. return false, NodeInfoResponse{} } + if recordID == "" || recordID != m.mesh.PeerRecordID(result.HostUUID) { + return false, NodeInfoResponse{} + } aged := (noderec.NodeInfo{ TelemetryValid: result.TelemetryValid, MSSince: result.MSSince, }).Aged(time.Since(start)) result.MSSince = aged.MSSince + result.discoveryRecordID = recordID + result.receivedAt = time.Now() slog.Debug("manual probe node-info up", "addr", addr, "port", port, "gpus", len(result.GPUs), "has_cpu", result.CPU != nil, "has_memory", result.Memory != nil, @@ -604,21 +692,20 @@ func (m *Manager) addNode(entry ManualEntry) ManualNodeStatus { MTLSRequired: entry.TLSPort > 0 && entry.MTLS, } + tracked := &trackedNode{entry: entry, status: status} m.mu.Lock() - m.nodes[id] = &trackedNode{entry: entry, status: status} + m.nodes[id] = tracked m.mu.Unlock() go func() { m.probeNode(entry) - m.mu.RLock() + m.mu.Lock() + defer m.mu.Unlock() tn, exists := m.nodes[id] - if !exists { - m.mu.RUnlock() + if !exists || tn != tracked { return } - status = tn.status - m.mu.RUnlock() - m.codec.Notify("node/discovered", status) + m.codec.Notify("node/discovered", tn.statusAt(time.Now())) }() return status @@ -626,11 +713,11 @@ func (m *Manager) addNode(entry ManualEntry) ManualNodeStatus { func (m *Manager) removeNode(id string) bool { m.mu.Lock() + defer m.mu.Unlock() _, exists := m.nodes[id] if exists { delete(m.nodes, id) } - m.mu.Unlock() if exists { m.codec.Notify("node/removed", ManualNodeStatus{ID: id}) @@ -649,11 +736,12 @@ func (m *Manager) removeNode(id string) bool { } func (m *Manager) listNodes() []ManualNodeStatus { + m.reconcileTrust() m.mu.RLock() defer m.mu.RUnlock() result := make([]ManualNodeStatus, 0, len(m.nodes)) for _, tn := range m.nodes { - result = append(result, tn.status) + result = append(result, tn.statusAt(time.Now())) } return result } diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index 496d1fe62..0aab40c5b 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -475,6 +475,7 @@ func TestProbeNode_DiscardsInvalidAuthenticatedHardware(t *testing.T) { f.manager.probeNode(f.entry) initial := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, f.frames, methodIs("node/updated"))) require.Equal(t, noderec.IdentityPinned, initial.IdentityVerification) + require.NotEmpty(t, initial.DiscoveryRecordID) require.True(t, initial.NodeInfoUp) require.NotEmpty(t, initial.GPUs) require.NotNil(t, initial.CPU) @@ -496,6 +497,7 @@ func TestProbeNode_DiscardsInvalidAuthenticatedHardware(t *testing.T) { } updated := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, f.frames, methodIs("node/updated"))) assert.Equal(t, noderec.IdentityUnverified, updated.IdentityVerification) + assert.Empty(t, updated.DiscoveryRecordID) assert.Empty(t, updated.HostUUID) assert.False(t, updated.NodeInfoUp) assert.Empty(t, updated.GPUs) @@ -528,13 +530,15 @@ func TestProbeNode_DiscardsInvalidAuthenticatedHardware(t *testing.T) { const manualPeerUUID = "peer" type manualPeerFixture struct { - manager *Manager - peerDir string - frames *captureRW - entry ManualEntry - info NodeInfoResponse - server *httptest.Server - respond func(http.ResponseWriter) + manager *Manager + selfDir string + peerDir string + frames *captureRW + entry ManualEntry + info NodeInfoResponse + server *httptest.Server + beforeResponse func() + respond func(http.ResponseWriter) } // Create isolated cluster identities pinned in both directions and a local @@ -568,6 +572,7 @@ func newManualPeerFixture(t *testing.T) *manualPeerFixture { t.Cleanup(manager.peerClients.CloseIdle) f := &manualPeerFixture{ manager: manager, + selfDir: clientDir, peerDir: serverDir, frames: frames, info: sampleInfo(), @@ -581,6 +586,9 @@ func newManualPeerFixture(t *testing.T) *manualPeerFixture { http.Error(w, "forbidden", http.StatusForbidden) return } + if f.beforeResponse != nil { + f.beforeResponse() + } if f.respond != nil { f.respond(w) return @@ -603,6 +611,129 @@ func newManualPeerFixture(t *testing.T) *manualPeerFixture { return f } +func TestProbeNode_DiscardsResultsAfterIdentityOrEntryChanges(t *testing.T) { + test := func(name string, change func(*testing.T, *manualPeerFixture)) { + t.Run(name, func(t *testing.T) { + f := newManualPeerFixture(t) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + f.beforeResponse = func() { close(entered); <-release } + go func() { f.manager.probeNode(f.entry); close(done) }() + select { + case <-entered: + case <-time.After(3 * time.Second): + unblock() + t.Fatal("authenticated request did not reach the server") + } + change(t, f) + unblock() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("probe did not finish") + } + + status := f.manager.listNodes() + require.Len(t, status, 1) + assert.NotEqual(t, noderec.IdentityPinned, status[0].IdentityVerification) + assert.False(t, status[0].TelemetryValid) + assert.Empty(t, status[0].HostUUID) + }) + } + test("local certificate replaced during read", func(t *testing.T, f *manualPeerFixture) { + clustertrusttest.WriteKeypair(t, f.selfDir, "self") + }) + test("local node readmitted during read", func(t *testing.T, f *manualPeerFixture) { + clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) + }) + test("peer pin removed during read", func(t *testing.T, f *manualPeerFixture) { + clustertrusttest.RemovePeerPin(t, f.selfDir, manualPeerUUID) + }) + test("peer pin replaced during read", func(t *testing.T, f *manualPeerFixture) { + clustertrusttest.WritePeerPin(t, f.selfDir, manualPeerUUID) + }) + test("entry replaced under the same ID during read", func(t *testing.T, f *manualPeerFixture) { + f.manager.mu.Lock() + defer f.manager.mu.Unlock() + // Removing and re-adding the same alias creates a new tracked entry. + f.manager.nodes[nodeID(f.entry)] = &trackedNode{entry: f.entry} + }) +} + +func TestProbeNode_DiscardsInFlightHardwareAfterPeerReadmission(t *testing.T) { + // A new admission must reject an earlier response even when its certificate + // is unchanged. Inspect the emitted update before cached reads reconcile trust. + f := newManualPeerFixture(t) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + f.beforeResponse = func() { close(entered); <-release } + go func() { f.manager.probeNode(f.entry); close(done) }() + select { + case <-entered: + case <-time.After(3 * time.Second): + unblock() + t.Fatal("authenticated request did not reach the server") + } + pinPath := filepath.Join(f.selfDir, "trusted", manualPeerUUID+".json") + data, err := os.ReadFile(pinPath) + require.NoError(t, err) + var pin map[string]json.RawMessage + require.NoError(t, json.Unmarshal(data, &pin)) + pin["admissionEpoch"] = json.RawMessage(`2`) + pin["pinnedAt"] = json.RawMessage(`1000`) + data, err = json.Marshal(pin) + require.NoError(t, err) + require.NoError(t, os.WriteFile(pinPath, data, 0o600)) + unblock() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("probe did not finish") + } + + updated := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, f.frames, methodIs("node/updated"))) + assert.Equal(t, noderec.IdentityUnverified, updated.IdentityVerification) + assert.Empty(t, updated.DiscoveryRecordID) + assert.Empty(t, updated.HostUUID) + assert.False(t, updated.NodeInfoUp) + assert.Empty(t, updated.GPUs) + assert.Nil(t, updated.CPU) + assert.Nil(t, updated.Memory) + assert.False(t, updated.TelemetryValid) + assert.Zero(t, updated.MSSince) +} + +func TestListNodes_DiscardsHardwareAfterPinRemoval(t *testing.T) { + f := newManualPeerFixture(t) + f.manager.probeNode(f.entry) + + clustertrusttest.RemovePeerPin(t, f.selfDir, manualPeerUUID) + status := f.manager.listNodes() + + require.Len(t, status, 1) + assert.Equal(t, noderec.IdentityUnverified, status[0].IdentityVerification) + assert.False(t, status[0].NodeInfoUp) + assert.Empty(t, status[0].GPUs) + assert.False(t, status[0].TelemetryValid) +} + +func TestListNodes_PreservesTimeElapsedSinceHardwareProbe(t *testing.T) { + f := newManualPeerFixture(t) + f.manager.probeNode(f.entry) + first := f.manager.listNodes() + require.Len(t, first, 1) + time.Sleep(25 * time.Millisecond) + + second := f.manager.listNodes() + + require.Len(t, second, 1) + assert.GreaterOrEqual(t, second[0].MSSince, first[0].MSSince+25) +} + // Probe a mutually pinned peer and check that its hardware and host identity // enter the manual status together, with sample age retained across the fetch. func TestProbeNode_AcceptsMeasurementsFromPinnedPeer(t *testing.T) { diff --git a/services/nvpair-node-info/main_test.go b/services/nvpair-node-info/main_test.go index 47737af1e..efc155a43 100644 --- a/services/nvpair-node-info/main_test.go +++ b/services/nvpair-node-info/main_test.go @@ -28,6 +28,7 @@ import ( "nvpair-shared/applog" "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/jsonrpc" "nvpair-shared/noderec" ) @@ -85,6 +86,45 @@ func TestNodeInfo_StreamsPrivateSnapshotsAndAcceptsMetadata(t *testing.T) { assert.Nil(t, public.Peer) } +// Drive membership on disk while a real owned worker is running. Its private +// stream and public hardware display must work before join and after leave. +func TestNodeInfo_PrivateIPCContinuesAcrossJoinAndLeave(t *testing.T) { + const hostUUID = "owned-host" + dir := t.TempDir() + process := startNodeInfoForTest(t, hostUUID, "--cluster-dir", dir) + initial := process.nextSnapshot(t) + assert.True(t, initial.ClusterUUID == nil || *initial.ClusterUUID == "") + + clustertrusttest.Join(t, dir, "test-cluster", hostUUID) + var joined noderec.NodeInfo + for joined.ClusterUUID == nil || *joined.ClusterUUID != hostUUID { + joined = process.nextSnapshot(t) + } + // A polling reader can still hold the admission file open during departure. + // Clear active membership in place so departure also works with that reader. + admissionReader, err := os.Open(filepath.Join(dir, "admission.json")) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, admissionReader.Close()) }) + clustertrusttest.WriteAdmission(t, dir, "", 0) + left := process.nextSnapshot(t) + for left.ClusterUUID != nil && *left.ClusterUUID != "" { + left = process.nextSnapshot(t) + } + assert.Equal(t, initial.HostUUID, left.HostUUID) + assert.NotNil(t, left.GPUs) + + client := &http.Client{Timeout: 2 * time.Second} + t.Cleanup(client.CloseIdleConnections) + response, err := client.Get(process.inventoryURL) + require.NoError(t, err) + defer func() { assert.NoError(t, response.Body.Close()) }() + var public noderec.NodeInfo + require.NoError(t, json.NewDecoder(response.Body).Decode(&public)) + assert.Equal(t, http.StatusOK, response.StatusCode) + assert.Equal(t, hostUUID, public.HostUUID) + assert.Nil(t, public.Peer) +} + // nodeInfoTestProcess is the test's broker-side connection to a real child. type nodeInfoTestProcess struct { stdin io.Writer diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index f8f2b7e22..3921a5ded 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -20,6 +20,8 @@ Concretely, the daemon: `pinned` peer record, replacing only that successful discovery. Failed candidates stay separate and `unverified`. Public HTTP is never a scanner measurement source, and operator CAs cannot substitute for cluster pins. + Verified record IDs are opaque and change with the local certificate or + admission and the peer's exact pin or admission. Host UUIDs remain stable. - **Refreshes models** every 15 seconds from verified endpoints, preserving the last successful inventory during outages. A newly authenticated peer schedules a model refresh immediately. Remote model lists use pinned diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 50fd83c3e..9a6dedf54 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -4,7 +4,6 @@ package main import ( - "bytes" "context" "crypto/x509" "encoding/json" @@ -421,10 +420,10 @@ type discoveryCandidate struct { } type peerInfoSample struct { - info NodeInfoResponse - certificate *x509.Certificate - localCertificate []byte - receivedAt time.Time + info NodeInfoResponse + certificate *x509.Certificate + recordID string + receivedAt time.Time } func newNodeInfoClients(mesh *clustertrust.Mesh) *clustertrust.PeerClientPool { @@ -526,10 +525,6 @@ func (d *daemon) readPeerInfo(ctx context.Context, node noderec.DirectoryNode) ( if port < 1 || port > 65535 { return peerInfoSample{}, false } - config, ok := d.mesh.ClientTLSConfig(principal) - if !ok { - return peerInfoSample{}, false - } ask := func(host string) (peerInfoSample, bool) { started := time.Now() origin := "https://" + net.JoinHostPort(host, strconv.Itoa(port)) @@ -546,6 +541,7 @@ func (d *daemon) readPeerInfo(ctx context.Context, node noderec.DirectoryNode) ( defer release() requestCtx, cancel := context.WithTimeout(ctx, nodeInfoFetchTimeout) defer cancel() + recordID := d.mesh.PeerRecordID(principal) client, ok := d.nodeInfoClients.Client(principal) if !ok { return peerInfoSample{}, false @@ -576,7 +572,7 @@ func (d *daemon) readPeerInfo(ctx context.Context, node noderec.DirectoryNode) ( if info.HostUUID != principal || info.ClusterUUID == nil || *info.ClusterUUID != principal || !validPeerMetadata(info.Peer) { return peerInfoSample{}, false } - sample := peerInfoSample{info: info.Aged(time.Since(started)), certificate: resp.TLS.PeerCertificates[0], localCertificate: config.Certificates[0].Certificate[0], receivedAt: time.Now()} + sample := peerInfoSample{info: info.Aged(time.Since(started)), certificate: resp.TLS.PeerCertificates[0], recordID: recordID, receivedAt: time.Now()} if !d.currentPin(sample) { return peerInfoSample{}, false } @@ -610,8 +606,7 @@ func validPeerMetadata(peer *noderec.PeerMetadata) bool { func (d *daemon) currentPin(sample peerInfoSample) bool { d.mesh.Refresh() - config, ok := d.mesh.ClientTLSConfig(sample.info.HostUUID) - return ok && bytes.Equal(sample.localCertificate, config.Certificates[0].Certificate[0]) && d.mesh.VerifyPeerCertificate(sample.info.HostUUID, sample.certificate) + return sample.recordID != "" && sample.recordID == d.mesh.PeerRecordID(sample.info.HostUUID) && d.mesh.VerifyPeerCertificate(sample.info.HostUUID, sample.certificate) } // currentPeerResult also rejects results for an endpoint replaced while a read @@ -643,7 +638,7 @@ func (d *daemon) publishPeerInfo(candidateID string, sample peerInfoSample) { services[key] = noderec.ServiceStatus{Port: port} } node := noderec.DirectoryNode{ - DiscoveryRecordID: "peer:" + info.HostUUID, IdentityVerification: noderec.IdentityPinned, + DiscoveryRecordID: sample.recordID, IdentityVerification: noderec.IdentityPinned, HostUUID: info.HostUUID, ClusterUUID: *info.ClusterUUID, Trusted: true, Name: info.Peer.Name, IP: info.Peer.Addresses[0], IPs: info.Peer.Addresses, Services: services, GPUs: info.GPUs, CPU: info.CPU, Memory: info.Memory, @@ -731,9 +726,9 @@ func (d *daemon) onBrowse(ev DiscoveryEvent) { } } if peer, present := d.dir.get(candidate.peerUUID); present { + d.forget(peer.HostUUID) d.dir.remove(peer.HostUUID) delete(d.peerCertificates, peer.HostUUID) - d.forget(peer.HostUUID) d.emit(noderec.NotifyNodeRemoved, peer) } } @@ -798,7 +793,8 @@ func (d *daemon) reconcileTrust() { d.peerMu.Lock() defer d.peerMu.Unlock() for uuid, cert := range d.peerCertificates { - if d.mesh.VerifyPeerCertificate(uuid, cert) { + node, present := d.dir.get(uuid) + if present && node.DiscoveryRecordID == d.mesh.PeerRecordID(uuid) && d.mesh.VerifyPeerCertificate(uuid, cert) { continue } delete(d.peerCertificates, uuid) @@ -809,8 +805,8 @@ func (d *daemon) reconcileTrust() { } } if node, ok := d.dir.get(uuid); ok { - d.dir.remove(uuid) d.forget(uuid) + d.dir.remove(uuid) d.emit(noderec.NotifyNodeRemoved, node) } } @@ -1210,7 +1206,7 @@ func (d *daemon) peerReachable(uuid string) bool { node, ok := d.dir.get(uuid) cert := d.peerCertificates[uuid] d.mesh.Refresh() - valid := ok && d.mesh.VerifyPeerCertificate(node.ClusterUUID, cert) + valid := ok && node.DiscoveryRecordID == d.mesh.PeerRecordID(uuid) && d.mesh.VerifyPeerCertificate(node.ClusterUUID, cert) d.peerMu.Unlock() if !valid { return false @@ -1498,9 +1494,9 @@ func (d *daemon) expireRetainedPeer(target noderec.DirectoryNode) { if since, ok := d.activitySince(peer.HostUUID); ok && since < activityFreshness { return } + d.forget(peer.HostUUID) d.dir.remove(peer.HostUUID) delete(d.peerCertificates, peer.HostUUID) - d.forget(peer.HostUUID) d.emit(noderec.NotifyNodeRemoved, peer) } @@ -1562,6 +1558,7 @@ func (d *daemon) refreshNodeModels(hostUUID, ip string, emPort int, clusterUUID // askRemembered retains a working address across model sweeps until it fails or // is withdrawn from the current record. func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []string, emPort int, clusterUUID string) bool { + original, _ := d.dir.get(hostUUID) if d.reg != nil && hostUUID == d.reg.record().HostUUID { hosts = []string{loopbackHost} clusterUUID = "" @@ -1580,6 +1577,8 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s d.mesh.Refresh() if target, present := d.dir.get(hostUUID); present && target.IdentityVerification == noderec.IdentityPinned { if target.ClusterUUID != clusterUUID || + target.DiscoveryRecordID != original.DiscoveryRecordID || + target.DiscoveryRecordID != d.mesh.PeerRecordID(clusterUUID) || !slices.Equal(target.CandidateIPs(), hosts) || !d.mesh.VerifyPeerCertificate(clusterUUID, inventory.certificate) || !d.mesh.VerifyPeerCertificate(clusterUUID, d.peerCertificates[hostUUID]) { @@ -1617,7 +1616,11 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s // re-discovery re-probes from scratch rather than resurrecting stale metrics. func (d *daemon) forget(hostUUID string) { d.telemetryRetries.forget(hostUUID) - d.enrichHosts.forget(hostKey{hostUUID: "peer:" + hostUUID, service: noderec.ServiceNodeInfo}) + if d.dir != nil { + if node, present := d.dir.get(hostUUID); present { + d.enrichHosts.forget(hostKey{hostUUID: node.DiscoveryRecordID, service: noderec.ServiceNodeInfo}) + } + } d.enrichHosts.forget(hostKey{hostUUID: hostUUID, service: noderec.ServiceEngineManager}) d.infoMu.Lock() delete(d.lastInfoAt, hostUUID) diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index 468de7e13..7832fbc83 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -467,7 +467,7 @@ func TestDiscovery_PublishesAuthenticatedHardwareSnapshots(t *testing.T) { } var snapshot hardwareSnapshot require.NoError(t, json.Unmarshal(message.Params, &snapshot)) - assert.Equal(t, "peer:peer-uuid", snapshot.DiscoveryRecordID) + assert.Equal(t, f.daemon.mesh.PeerRecordID("peer-uuid"), snapshot.DiscoveryRecordID) assert.Equal(t, noderec.IdentityPinned, snapshot.IdentityVerification) assert.Equal(t, f.info.GPUs, snapshot.Info.GPUs) snapshots++ @@ -789,24 +789,25 @@ func TestDiscovery_RejectsMissingOrInvalidPeerDetails(t *testing.T) { } func TestDiscovery_RejectsCertificatesThatDoNotMatchExpectedPeer(t *testing.T) { + const expectedPeerUUID = "peer-uuid" + test := func(name, presentedUUID string) { t.Run(name, func(t *testing.T) { - f := newDiscoveryPeerFixture(t, "peer-uuid") + f := newDiscoveryPeerFixture(t, expectedPeerUUID) wrongDir := t.TempDir() clustertrusttest.Join(t, wrongDir, "test-cluster", presentedUUID) - // A certificate saved for another peer cannot authenticate this peer. + // Keep the expected peer's original pin, even when the presented UUID + // is also trusted. Another trusted peer cannot authenticate this one. writePinFromCert(t, f.selfDir, presentedUUID, filepath.Join(wrongDir, "node.crt")) - if presentedUUID == "peer-uuid" { - writePinFromCert(t, f.selfDir, "peer-uuid", filepath.Join(f.peerDir, "node.crt")) - } + writePinFromCert(t, f.selfDir, expectedPeerUUID, filepath.Join(f.peerDir, "node.crt")) wrongMesh := clustertrust.Open(wrongDir) f.server.TLS.GetConfigForClient = wrongMesh.ServerTLSConfig().GetConfigForClient f.discover(t) - _, present := f.daemon.dir.get("peer-uuid") + _, present := f.daemon.dir.get(expectedPeerUUID) assert.False(t, present) }) } - test("new certificate for same UUID does not match saved certificate", "peer-uuid") + test("new certificate for same UUID does not match saved certificate", expectedPeerUUID) test("saved certificate belongs to another peer", "other-peer") } @@ -826,6 +827,9 @@ func TestDiscovery_DiscardsResultsInvalidatedDuringRead(t *testing.T) { require.NoError(t, os.Remove(filepath.Join(f.selfDir, "admission.json"))) }) test("local certificate replaced", func(t *testing.T, f *discoveryPeerFixture) { clustertrusttest.WriteKeypair(t, f.selfDir, "self-uuid") }) + test("local node readmitted with the same certificate", func(t *testing.T, f *discoveryPeerFixture) { + clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) + }) test("discovery removed", func(t *testing.T, f *discoveryPeerFixture) { f.daemon.onBrowse(DiscoveryEvent{Type: "removed", Node: f.raw}) }) @@ -860,15 +864,29 @@ func (f *discoveryPeerFixture) invalidateDuringRead(t *testing.T, invalidate fun func TestDiscovery_ReauthenticatesAfterSavedCertificateChanges(t *testing.T) { f := newDiscoveryPeerFixture(t, "peer-uuid") f.discover(t) + previous, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) clustertrusttest.WriteKeypair(t, f.peerDir, "peer-uuid") writePinFromCert(t, f.selfDir, "peer-uuid", filepath.Join(f.peerDir, "node.crt")) f.daemon.reloadTrust() - _, present := f.daemon.dir.get("peer-uuid") + _, present = f.daemon.dir.get("peer-uuid") assert.False(t, present) f.daemon.refreshCandidatesOnce(context.Background()) node, present := f.daemon.dir.get("peer-uuid") require.True(t, present) assert.True(t, node.Trusted) + assert.NotEqual(t, previous.DiscoveryRecordID, node.DiscoveryRecordID) +} + +func TestDiscovery_LocalCertificateReplacementDiscardsVerifiedInventory(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + + clustertrusttest.WriteKeypair(t, f.selfDir, "self-uuid") + f.daemon.reloadTrust() + + _, present := f.daemon.dir.get("peer-uuid") + assert.False(t, present) } type heldModelResponseTransport struct { @@ -1329,7 +1347,7 @@ func TestDiscovery_RetainedPeerExpiresAfterAvailabilityEvidenceEnds(t *testing.T var removed noderec.NodeEvent require.NoError(t, json.Unmarshal(message.Params, &removed)) assert.Equal(t, noderec.NotifyNodeRemoved, message.Method) - assert.Equal(t, "peer:peer-uuid", removed.Node.DiscoveryRecordID) + assert.Equal(t, f.daemon.mesh.PeerRecordID("peer-uuid"), removed.Node.DiscoveryRecordID) f.daemon.mesh.Refresh() assert.True(t, f.daemon.mesh.Clustered()) assert.True(t, f.daemon.mesh.HasPin("peer-uuid")) @@ -1601,3 +1619,22 @@ func TestDiscovery_DiscardsQueuedModelsForChangedAddresses(t *testing.T) { assert.Equal(t, before, after) assert.Empty(t, output.Bytes()) } + +func TestPeerReachable_RejectsCachedLivenessAfterReadmission(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.daemon.noteActivity("peer-uuid", 0) + require.True(t, f.daemon.peerReachable("peer-uuid")) + require.True(t, f.daemon.reachable(f.raw)) + + // Readmission changes the proof binding even when both certificates remain. + // Availability checks must reject old evidence before a reload notification. + clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) + + assert.False(t, f.daemon.peerReachable("peer-uuid")) + assert.False(t, f.daemon.reachable(f.raw)) + + f.daemon.reloadTrust() + f.daemon.refreshCandidatesOnce(context.Background()) + assert.True(t, f.daemon.reachable(f.raw)) +} diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 5c1f736c2..87acdf0fc 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -87,6 +87,13 @@ membership, or registrations change, and is replayed after node-info restarts. Remote discovery claims cannot populate this metadata. These links are internal worker IPC; desktop and TUI continue using public HTTP for hardware displays. +Verified record IDs bind scanner and manual proofs to current pins and local +membership. Trust changes withdraw affected records, full hardware snapshots, +and scheduling samples from both sources. Queued events and scheduler replay +recheck that binding. A scanner crash withdraws its entire directory; its +replacement authenticates remote records again while local IPC replays retain +their age. Notifications from a previous worker cannot restore its records. + ## CLI Flags | Flag | Default | Description | @@ -575,6 +582,10 @@ Manually added nodes also surface in the shared `discovery:get-nodes` / `discove **Manual → proxy bridge.** When the broker supervises both `nvpair-manual-nodes` and a proxy, it also bridges a manual node whose engine is reachable into that proxy via `node/add-manual` (host/port from the node's per-engine status), so inference can route to it through `ollama-proxy:node/select` / `lmstudio-proxy:node/select` just like a relay-discovered node. This is per-engine: a node whose `ollama_*` status is up is bridged into `ollama-proxy` (host/port from `ollama_port`), and one whose `lmstudio_*` status is up into `lmstudio-proxy` (from `lmstudio_port`) — a node running both is bridged into both. Manual nodes are by definition the ones that never appear via the daemon's `_nvpair-node` discovery, so this explicit add is what makes them routable. The bridge tracks reachability: an engine that goes down (or a node that is removed, or whose prober crashes) is pulled back out with `node/remove-manual`. A proxy that isn't supervised → that leg is a no-op; manual nodes still appear in the discovery snapshot as before. +Each engine has one ordered bridge sender outside the discovery lock. It keeps the latest desired target per key, coalescing intermediate status updates while the proxy is slow. Withdrawals follow any in-flight add, and a later status update retries a failed add or removal. A replacement proxy receives the current targets after its facades are enabled; a canceled configuration is not replayed. Manual bridge calls therefore do not hold the discovery lock while waiting for a proxy. + +Proxy discovery subscriptions likewise queue their latest complete snapshot per engine. One sender per proxy process writes these notifications outside the discovery and relay locks, preserving empty withdrawals and replacing pending snapshots from retired subscriptions. A blocked proxy therefore cannot hold those locks through either its manual bridge or its discovery subscription. + #### `cluster:` / `nodes:` (generic relay) Any request whose method starts with `cluster:` or `nodes:` is **forwarded to `nvpair-cluster-manager`** verbatim (no prefix stripping — these are the manager's own native namespaces), and the manager's result or JSON-RPC error is relayed straight back to the caller unchanged. As with the proxy relay this means the cluster-manager's whole surface is reachable through the broker without enumerating individual methods. Current methods include `cluster:get-node-id`, `cluster:set-identity`, `cluster:create`, `cluster:invite-node`, `cluster:respond-to-invite`, `cluster:cancel-invite`, `cluster:invite-status`, `nodes:get-initial`, and `nodes:remove` (see the [`nvpair-cluster-manager` README](../nvpair-cluster-manager/README.md) for their params/results and app-defined error codes). diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 72b9e4cac..b9a5c2848 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -20,6 +20,7 @@ import ( "nvpair-shared/appdir" "nvpair-shared/applog" + "nvpair-shared/clustertrust" "nvpair-shared/engines" "nvpair-shared/errors" "nvpair-shared/nodeid" @@ -179,6 +180,13 @@ type Broker struct { clusterMgrPath string schedulerPath string clusterDir string + // discoveryMu serializes worker events with trust invalidation and crash + // cleanup, so a queued event cannot restore a withdrawn source. + discoveryMu sync.Mutex + discoveryMesh *clustertrust.Mesh + discoveryMeshOnce sync.Once + manualGeneration uint64 // protected by discoveryMu + nodeInfoGeneration uint64 // protected by discoveryMu // Managed-port state is prepared before proxy startup and read by the proxy // supervisor/reader goroutines. Ollama commits its pending backend move after // its proxy reserves :11434; LM Studio moves through engine-manager first, @@ -624,6 +632,8 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { b.scannerPath, applog.LevelString(), b.store, + &b.discoveryMu, + b.currentDiscoveryMesh(), b.relayDir, b.failWorkloadsForNode, func(telemetry noderec.NodeTelemetry) { b.ingestTelemetry(sourceScanner, telemetry) }, @@ -660,7 +670,10 @@ func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { // One port serves public display HTTP and pinned peer mTLS. Local service // measurements still use the owned worker's stdio, independent of membership. extra := append(b.clusterDirArgs(), "--node-id", b.nodeID) - np, err := startNodeInfo(b.nodeInfoPath, applog.LevelString(), b.forwardNodeInfoNotification, extra...) + b.discoveryMu.Lock() + generation := b.nodeInfoGeneration + b.discoveryMu.Unlock() + np, err := startNodeInfo(b.nodeInfoPath, applog.LevelString(), b.nodeInfoNotificationHandler(generation), extra...) if err != nil { return nil, err } @@ -674,6 +687,24 @@ func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { return np, nil } +func (b *Broker) nodeInfoNotificationHandler(generation uint64) func(string, json.RawMessage) { + return func(method string, params json.RawMessage) { + receivedAt := time.Now() + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + if generation == b.nodeInfoGeneration { + b.forwardNodeInfoNotification(method, params, receivedAt) + } + } +} + +func (b *Broker) clearNodeInfoState() { + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + b.nodeInfoGeneration++ + b.setNodeInfo(nil) +} + // proxyArgs builds the child's argv. Only process-scoped flags remain: which // engines it fronts, on which ports, and with which alias addresses all travel // in facade/enable, because the broker plans a different port per engine and a @@ -1175,7 +1206,10 @@ func (b *Broker) spawnEngineMgr() (supervisedHandle, error) { } func (b *Broker) spawnManualNodes() (supervisedHandle, error) { - w, err := startRPCWorker("manual-nodes", b.manualNodesPath, append(b.logLevelArgs(), b.clusterDirArgs()...), b.forwardManualNodesNotification) + b.discoveryMu.Lock() + generation := b.manualGeneration + b.discoveryMu.Unlock() + w, err := startRPCWorker("manual-nodes", b.manualNodesPath, append(b.logLevelArgs(), b.clusterDirArgs()...), b.manualNotificationHandler(generation)) if err != nil { return nil, err } @@ -1185,6 +1219,17 @@ func (b *Broker) spawnManualNodes() (supervisedHandle, error) { return w, nil } +func (b *Broker) manualNotificationHandler(generation uint64) func(string, json.RawMessage) { + return func(method string, params json.RawMessage) { + receivedAt := time.Now() + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + if generation == b.manualGeneration { + b.handleManualNodesNotification(method, params, receivedAt) + } + } +} + func (b *Broker) spawnSettings() (supervisedHandle, error) { w, err := startRPCWorker("settings", b.settingsPath, b.logLevelArgs(), b.forwardSettingsNotification) if err != nil { @@ -1200,6 +1245,7 @@ func (b *Broker) spawnJobScheduler() (supervisedHandle, error) { if err != nil { return nil, err } + b.reconcileDiscoveryTrust() // workloadEmitMu prevents an accepted workload transition from interleaving // with its baseline. schedulerFeedMu also holds discovery fanout until the @@ -1209,7 +1255,7 @@ func (b *Broker) spawnJobScheduler() (supervisedHandle, error) { b.setScheduler(w) replayed := b.replayActiveWorkloadsToScheduler(w) telemetryReplayed := b.replayTelemetryToScheduler(w) - if err := w.Notify("discovery:nodes-changed", b.store.SchedulingSnapshot()); err != nil { + if err := w.Notify("discovery:nodes-changed", b.schedulingSnapshot()); err != nil { slog.Warn("push discovery baseline to scheduler failed", "err", err) } b.schedulerFeedMu.Unlock() @@ -1439,13 +1485,15 @@ func resolveLocalNodeID(clusterDir string) string { // the rest of its life — in the roster and ranked by the scheduler, yet // exchanging no cluster traffic. // -// Scanner updates its local announcement immediately. Node-info reads the shared -// cluster directory itself. The scanner write is asynchronous because this is -// cluster-manager's reader. +// Withdraw obsolete proofs before accepting more worker events. Scanner also +// updates its local announcement immediately. Node-info reads the shared cluster +// directory itself. +// Child writes run asynchronously because this is cluster-manager's reader. func (b *Broker) applyClusterIdentityChange() { if b.clusterDir == "" { return } + b.reconcileDiscoveryTrust() slog.Info("cluster identity changed; converging this node's advertised cluster identity") if sc := b.getScanner(); sc != nil { go sc.reloadIdentity() @@ -1556,6 +1604,15 @@ func (b *Broker) clearEngineRelaySub() { } func (b *Broker) forwardManualNodesNotification(method string, params json.RawMessage) { + receivedAt := time.Now() + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + b.handleManualNodesNotification(method, params, receivedAt) +} + +// handleManualNodesNotification runs with discoveryMu held, including when a +// worker generation check has already been made by the spawn callback. +func (b *Broker) handleManualNodesNotification(method string, params json.RawMessage, receivedAt time.Time) { if b.dispatchErrorsNotif("manual-nodes", method, params) { return } @@ -1573,7 +1630,14 @@ func (b *Broker) forwardManualNodesNotification(method string, params json.RawMe slog.Warn("manual-nodes node payload missing id", "method", method) return } - b.upsertManualNode(s) + if s.IdentityVerification == noderec.IdentityPinned { + mesh := b.currentDiscoveryMesh() + mesh.Refresh() + if s.DiscoveryRecordID == "" || s.DiscoveryRecordID != mesh.PeerRecordID(s.HostUUID) { + s = s.withoutVerification() + } + } + b.upsertManualNodeAt(s, receivedAt) slog.Debug("manual node event", "event", method, "id", s.ID) case "node/removed": var s manualNodeStatus @@ -1602,9 +1666,13 @@ func (b *Broker) forwardManualNodesNotification(method string, params json.RawMe // the alias's key changed (node-info revealed its real UUID) the // old key is reprojected from a surviving alias or released. func (b *Broker) upsertManualNode(s manualNodeStatus) { + b.upsertManualNodeAt(s, time.Now()) +} + +// upsertManualNodeAt retains worker receipt time across discovery processing. +func (b *Broker) upsertManualNodeAt(s manualNodeStatus, receivedAt time.Time) { en := manualToEnriched(s) key := en.storeKey() - receivedAt := time.Now() b.manualMu.Lock() oldKey, existed := b.manualNodeKeys[s.ID] @@ -1612,6 +1680,11 @@ func (b *Broker) upsertManualNode(s manualNodeStatus) { b.manualNodeStatuses[s.ID] = manualNodeStatusEntry{status: s, receivedAt: receivedAt} b.manualMu.Unlock() + // Discard the old measurement before the replacement record becomes visible + // to scheduler replay, which checks authorization against the current record. + if previous, present := b.store.sourceNode(key, sourceManual); present && previous.DiscoveryRecordID != en.DiscoveryRecordID { + b.removeTelemetry(sourceManual, key) + } b.store.Upsert(en, sourceManual) if s.IdentityVerification == noderec.IdentityPinned && s.HostUUID != "" { b.ingestTelemetryAt(sourceManual, manualNodeTelemetry(s, s.HostUUID), receivedAt) @@ -1661,6 +1734,14 @@ func (b *Broker) reprojectOrRelease(key string) { survivor, ok := b.survivingAliasLocked(key) b.manualMu.Unlock() if ok { + if b.clusterDir != "" && survivor.status.IdentityVerification == noderec.IdentityPinned { + mesh := b.currentDiscoveryMesh() + mesh.Refresh() + if survivor.status.DiscoveryRecordID == "" || survivor.status.DiscoveryRecordID != mesh.PeerRecordID(survivor.status.HostUUID) { + b.upsertManualNode(survivor.status.withoutVerification()) + return + } + } b.store.Upsert(manualToEnriched(survivor.status), sourceManual) b.bridgeManualNode(survivor.status, key) if survivor.status.IdentityVerification == noderec.IdentityPinned { @@ -1668,9 +1749,13 @@ func (b *Broker) reprojectOrRelease(key string) { } return } + previous, present := b.store.sourceNode(key, sourceManual) b.store.Remove(key, sourceManual) b.removeManualNodeFromProxies(key) b.removeTelemetry(sourceManual, key) + if present { + b.failWorkloadsForWithdrawnPeer(previous) + } } // survivingAliasLocked returns the last-seen status of a manual alias still @@ -1694,6 +1779,9 @@ func (b *Broker) survivingAliasLocked(key string) (manualNodeStatusEntry, bool) // the discovery store. The durable broker-owned list is left intact and is // replayed into the replacement worker after it starts. func (b *Broker) clearManualNodesState() { + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + b.manualGeneration++ b.setManualNodes(nil) b.manualMu.Lock() keys := b.manualNodeKeys @@ -1708,12 +1796,16 @@ func (b *Broker) clearManualNodesState() { } seen[key] = true // Drop only the manual claim; a co-located scanner node keeps its record. + previous, present := b.store.sourceNode(key, sourceManual) b.store.Remove(key, sourceManual) b.removeTelemetry(sourceManual, key) // Pull the now-orphaned node out of every proxy too, so inference // doesn't keep a stale manual target while the replacement worker // restores and re-probes the durable entry. b.removeManualNodeFromProxies(key) + if present { + b.failWorkloadsForWithdrawnPeer(previous) + } } } @@ -1767,9 +1859,9 @@ func (b *Broker) forwardSchedulerNotification(method string, params json.RawMess // forwardNodeInfoNotification routes owned hardware snapshots and observed // addresses to scanner from node-info's stdout reader. Hardware supplies local // service measurements; observed addresses help rank this host's interfaces. -func (b *Broker) forwardNodeInfoNotification(method string, params json.RawMessage) { +func (b *Broker) forwardNodeInfoNotification(method string, params json.RawMessage, receivedAt time.Time) { if method == noderec.NotifyInfoUpdated { - b.acceptLocalInfo(params) + b.acceptLocalInfo(params, receivedAt) return } if method != noderec.NotifyObservedAddresses { @@ -1913,6 +2005,7 @@ func (b *Broker) deliverPrioritySnapshot(generation uint64, deliver func(schedul // the handle — and a replay that misses the handle is dropped by the generation // check and never retried, leaving the new child with no ranking at all. func (b *Broker) replayProxyStateAfterSpawn() { + b.replayManualProxyTargets() // Abandoned once teardown starts, which the onSpawned field comment // requires of any callback that talks to the worker: this one calls the // proxy, and a teardown is about to close that worker's stdin. It runs on @@ -2080,7 +2173,7 @@ func (b *Broker) Serve(ctx context.Context) error { // it and (when auto-restart is enabled) respawns it on an unexpected // exit instead of leaving the discovery snapshot frozen. b.scannerSup = newSupervisor("scanner", defaultRestartPolicy(), b.spawnScanner) - b.scannerSup.onCrash, b.scannerSup.onRecovered = b.supervisedWorkerCallbacks("scanner", func() { b.setScanner(nil) }) + b.scannerSup.onCrash, b.scannerSup.onRecovered = b.supervisedWorkerCallbacks("scanner", b.clearScannerState) if err := b.scannerSup.Start(); err != nil { return fmt.Errorf("start scanner: %w", err) } @@ -2133,7 +2226,7 @@ func (b *Broker) Serve(ctx context.Context) error { // than tearing down discovery. if b.nodeInfoPath != "" { b.nodeInfoSup = newSupervisor("node-info", defaultRestartPolicy(), b.spawnNodeInfo) - b.nodeInfoSup.onCrash, b.nodeInfoSup.onRecovered = b.supervisedWorkerCallbacks("node-info", func() { b.setNodeInfo(nil) }) + b.nodeInfoSup.onCrash, b.nodeInfoSup.onRecovered = b.supervisedWorkerCallbacks("node-info", b.clearNodeInfoState) if err := b.nodeInfoSup.Start(); err != nil { slog.Warn("node-info failed to start; continuing without local node advertisement", "path", b.nodeInfoPath, "err", err) b.nodeInfoSup = nil @@ -2358,11 +2451,30 @@ func (b *Broker) fanDiscoveryToScheduler() { if sc == nil { return } - if err := sc.Notify("discovery:nodes-changed", b.store.SchedulingSnapshot()); err != nil { + if err := sc.Notify("discovery:nodes-changed", b.schedulingSnapshot()); err != nil { slog.Warn("fan discovery:nodes-changed to scheduler failed", "err", err) } } +// schedulingSnapshot checks authorization at send time, including when a +// scheduler starts before the cluster-manager's trust notification arrives. +func (b *Broker) schedulingSnapshot() []AvailableNode { + nodes := b.store.SchedulingSnapshot() + if b.clusterDir == "" { + return nodes + } + mesh := b.currentDiscoveryMesh() + mesh.Refresh() + current := make([]AvailableNode, 0, len(nodes)) + for _, node := range nodes { + if node.IdentityVerification == noderec.IdentityPinned && (node.DiscoveryRecordID == "" || node.DiscoveryRecordID != mesh.PeerRecordID(node.HostUUID)) { + continue + } + current = append(current, node) + } + return current +} + // pushNodesChanged takes a fresh snapshot and writes it to the peer as a // bare-array discovery:nodes-changed notification — the payload shape is // intentionally just []AvailableNode, not wrapped, per the external @@ -2752,15 +2864,105 @@ func (b *Broker) forwardClusterManagerNotification(method string, params json.Ra } } -// applyClusterTrustChange asks scanner to rederive trust after the pin set changes. -// Node-info follows live membership from the same cluster directory. +// applyClusterTrustChange withdraws obsolete scanner and manual proofs, then +// prompts scanner to reauthenticate matching candidates. Node-info follows live +// membership from the same cluster directory. func (b *Broker) applyClusterTrustChange() { slog.Info("cluster pin set changed; re-deriving peer trust") + b.reconcileDiscoveryTrust() if sc := b.getScanner(); sc != nil { go sc.reloadTrust() } } +// currentDiscoveryMesh retains one live view of trust so temporary read +// failures cannot discard previously readable pins. +func (b *Broker) currentDiscoveryMesh() *clustertrust.Mesh { + b.discoveryMeshOnce.Do(func() { + if b.discoveryMesh == nil { + b.discoveryMesh = clustertrust.Open(b.clusterDir) + } + }) + return b.discoveryMesh +} + +func (b *Broker) reconcileDiscoveryTrust() { + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + mesh := b.currentDiscoveryMesh() + mesh.Refresh() + for _, node := range b.store.sourceNodes(sourceScanner) { + if node.IdentityVerification == noderec.IdentityPinned && node.DiscoveryRecordID != mesh.PeerRecordID(node.HostUUID) { + b.removeScannerNode(node) + } + } + b.manualMu.Lock() + var invalid []manualNodeStatus + for _, entry := range b.manualNodeStatuses { + status := entry.status + if status.IdentityVerification == noderec.IdentityPinned && (status.DiscoveryRecordID == "" || status.DiscoveryRecordID != mesh.PeerRecordID(status.HostUUID)) { + invalid = append(invalid, status.withoutVerification()) + } + } + b.manualMu.Unlock() + for _, status := range invalid { + b.upsertManualNode(status) + } +} + +// failWorkloadsForWithdrawnPeer checks the old record against live trust and any +// surviving current record. A failed probe or worker crash alone must not retire +// jobs. Callers hold discoveryMu so a new authenticated record cannot race this +// decision. +func (b *Broker) failWorkloadsForWithdrawnPeer(previous EnrichedNode) { + if b.workloads == nil || b.clusterDir == "" || previous.IdentityVerification != noderec.IdentityPinned { + return + } + mesh := b.currentDiscoveryMesh() + mesh.Refresh() + currentRecordID := mesh.PeerRecordID(previous.HostUUID) + if previous.DiscoveryRecordID == currentRecordID { + return + } + for _, source := range []nodeSource{sourceScanner, sourceManual} { + node, present := b.store.sourceNode(previous.HostUUID, source) + if present && (node.IdentityVerification == noderec.IdentityLocal || + (node.IdentityVerification == noderec.IdentityPinned && node.Trusted && currentRecordID != "" && node.DiscoveryRecordID == currentRecordID)) { + return + } + } + b.failWorkloadsForNode(previous.HostUUID, previous.ID) +} + +// clearScannerState withdraws the crashed worker's complete directory. Its +// replacement must authenticate remote records again; owned local inventory +// stays in the separate cache and is replayed with its original receipt time. +// A local scanner crash alone gives no evidence that a peer's workloads failed. +func (b *Broker) clearScannerState() { + b.discoveryMu.Lock() + defer b.discoveryMu.Unlock() + if scanner := b.getScanner(); scanner != nil { + scanner.stopped = true + } + b.setScanner(nil) + for _, node := range b.store.sourceNodes(sourceScanner) { + b.removeScannerNode(node) + } +} + +// removeScannerNode withdraws the scanner's record and measurements. Workloads +// are retired only if trust also changed and no current verified record survives. +func (b *Broker) removeScannerNode(node EnrichedNode) { + if b.relayDir != nil { + b.relayDir.Apply(noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: node.HostUUID, DiscoveryRecordID: node.DiscoveryRecordID}) + } + b.store.Remove(node.storeKey(), sourceScanner) + if node.IdentityVerification == noderec.IdentityPinned || node.IdentityVerification == noderec.IdentityLocal { + b.removeTelemetry(sourceScanner, node.HostUUID) + } + b.failWorkloadsForWithdrawnPeer(node) +} + // forwardWorkloadManagerNotification is the hook startWorkloadManager // invokes on the manager's reader goroutine for every notification it emits // on stdout. The manager translates peer-origin lifecycle events into @@ -2902,17 +3104,12 @@ func (b *Broker) applyWorkloadEvent(method string, params json.RawMessage, infer } } -// failWorkloadsForNode marks every non-terminal workload pinned to a node that -// just dropped out of discovery as failed. A node leaves the directory only -// after the scanner's miss-threshold + TCP-probe liveness guard has given up on -// it — a full minute of unbroken silence, a failed probe on every scan of the -// last three quarters of it, and no inference traffic from the node in that time -// either, not a transient mDNS miss — so this is a strong -// "the node is really gone" signal. That node's own proxy can no -// longer emit a terminal workload:completed/errored (its process or host is -// gone), so the broker synthesizes a workloads:upsert transition to "failed" -// for each affected workload — clearing the stale "in progress" line while -// keeping the entry in clients' history (we upsert, never remove). +// failWorkloadsForNode marks non-terminal workloads for a node as failed after +// the scanner's liveness guard declares it offline or its cluster trust is +// withdrawn. Callers must not use a local scanner crash as evidence of peer loss. +// The broker may no longer receive terminal workload:completed/errored events, +// so it synthesizes a workloads:upsert transition to "failed" for each affected +// workload, keeping the entry in clients' history. // // A workload is affected if the departed node is either its origin // (originatedFrom — the node that submitted it) or its executor (scheduledOn — diff --git a/services/nvpair-ui-broker/broker_test.go b/services/nvpair-ui-broker/broker_test.go index c2346d6e1..70d34be8a 100644 --- a/services/nvpair-ui-broker/broker_test.go +++ b/services/nvpair-ui-broker/broker_test.go @@ -4,16 +4,100 @@ package main import ( + "bytes" "encoding/json" + "io" "net" + "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" + "nvpair-ui-broker/relay" + "nvpair-ui-broker/workloadstore" ) +func TestLocalNodeInfo_PreservesAgeAcrossDiscoveryLockWait(t *testing.T) { + broker := &Broker{nodeID: "self"} + handler := broker.nodeInfoNotificationHandler(0) + wait := holdDiscoveryNotification(t, &broker.discoveryMu, func() { + handler(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"self","telemetryValid":true,"msSince":137}}`)) + }) + + require.NotNil(t, broker.localInfo) + info := broker.localInfo.info.Aged(time.Since(broker.localInfo.receivedAt)) + assert.GreaterOrEqual(t, info.MSSince, int64(137)+wait.Milliseconds()) +} + +func TestManualTelemetry_IncludesDiscoveryLockWaitInSampleAge(t *testing.T) { + broker := newManualTestBroker() + broker.clusterDir = t.TempDir() + clustertrusttest.Join(t, broker.clusterDir, "cluster", "self", "peer") + mesh := clustertrust.Open(broker.clusterDir) + params, err := json.Marshal(manualNodeStatus{ + ID: "configured", Address: "192.0.2.20", HostUUID: "peer", + DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, + TelemetryValid: true, MSSince: 137, + }) + require.NoError(t, err) + handler := broker.manualNotificationHandler(0) + wait := holdDiscoveryNotification(t, &broker.discoveryMu, func() { + handler("node/updated", params) + }) + + samples := broker.telemetry.Snapshot(time.Now()) + require.Len(t, samples, 1) + assert.GreaterOrEqual(t, samples[0].MSSince, int64(137)+wait.Milliseconds()) +} + +// Hold a received notification behind discovery processing, as happens while a +// manual update waits for proxy replies. This wait must contribute to sample age. +func holdDiscoveryNotification(t *testing.T, mu *sync.Mutex, notify func()) time.Duration { + t.Helper() + const minimumWait = 50 * time.Millisecond + mu.Lock() + entered, done := make(chan struct{}), make(chan struct{}) + go func() { + close(entered) + notify() + close(done) + }() + <-entered + // Allow scheduling time between the start signal and the handler's receipt + // timestamp while still requiring a substantial, observable lock wait. + time.Sleep(2 * minimumWait) + mu.Unlock() + select { + case <-done: + case <-time.After(2 * time.Second): + require.FailNow(t, "notification did not finish after discovery processing resumed") + } + return minimumWait +} + +func TestFanDiscoveryToScheduler_ExcludesPeersWhosePinWasRemoved(t *testing.T) { + b := newManualTestBroker() + b.clusterDir = t.TempDir() + clustertrusttest.Join(t, b.clusterDir, "cluster", "self", "peer") + mesh := clustertrust.Open(b.clusterDir) + b.store.Upsert(EnrichedNode{HostUUID: "peer", DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, Trusted: true}, sourceScanner) + var output bytes.Buffer + b.setScheduler(&rpcWorker{peer: NewPeer(NewCodec(&output))}) + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + + b.fanDiscoveryToScheduler() + + message, err := NewCodec(&output).Read() + require.NoError(t, err) + var nodes []AvailableNode + require.NoError(t, json.Unmarshal(message.Params, &nodes)) + assert.Empty(t, nodes) +} + // Give the broker a verified peer, an unverified record claiming that peer's // UUID, and an explicitly configured manual target. Read the scheduler's actual // notification and check that it contains only the peer and manual target. @@ -64,3 +148,468 @@ func TestFanDiscoveryToScheduler_ExcludesUnverifiedDiscoveries(t *testing.T) { assert.Equal(t, verifiedName, nodes[1].ID) <-done } + +// Both discovery sources can own one peer. Removing its pin must discard both +// verified records and cached measurements, while preserving independent discovery claims and +// the user's explicit manual engine configuration under its own key. +func TestClusterTrustChange_DiscardsVerifiedRecordsAndMeasurements(t *testing.T) { + broker := newManualTestBroker() + broker.clusterDir = t.TempDir() + clustertrusttest.Join(t, broker.clusterDir, "cluster", "self", "peer") + broker.relayDir = relay.NewDirectory() + mesh := clustertrust.Open(broker.clusterDir) + peer := noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", + DiscoveryRecordID: mesh.PeerRecordID("peer"), + IdentityVerification: noderec.IdentityPinned, Trusted: true, + } + broker.store.Upsert(directoryToEnriched(peer), sourceScanner) + broker.relayDir.Apply(noderec.NotifyNodeDiscovered, peer) + broker.upsertManualNode(manualNodeStatus{ + ID: "configured", HostUUID: "peer", DiscoveryRecordID: peer.DiscoveryRecordID, + IdentityVerification: noderec.IdentityPinned, Address: "192.0.2.20", + OllamaUp: true, OllamaPort: 11434, TelemetryValid: true, + }) + broker.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", TelemetryValid: true}) + candidate := noderec.DirectoryNode{HostUUID: "peer", DiscoveryRecordID: "candidate", IdentityVerification: noderec.IdentityUnverified} + broker.store.Upsert(directoryToEnriched(candidate), sourceScanner) + broker.relayDir.Apply(noderec.NotifyNodeDiscovered, candidate) + principal := "peer" + require.True(t, broker.store.AcceptInfo(noderec.NodeInfoEvent{ + HostUUID: "peer", ClusterUUID: "peer", DiscoveryRecordID: peer.DiscoveryRecordID, + IdentityVerification: noderec.IdentityPinned, + Info: noderec.NodeInfo{HostUUID: "peer", ClusterUUID: &principal}, + }, time.Now())) + + clustertrusttest.RemovePeerPin(t, broker.clusterDir, "peer") + broker.applyClusterTrustChange() + + assert.Empty(t, broker.telemetry.Snapshot(time.Now())) + assert.False(t, broker.store.verifiedSource("peer", sourceScanner)) + assert.False(t, broker.store.verifiedSource("peer", sourceManual)) + assert.Empty(t, broker.manualAliasForKey("peer")) + assert.Equal(t, "configured", broker.manualAliasForKey("manual:configured")) + assert.Equal(t, []noderec.DirectoryNode{candidate}, broker.relayDir.Snapshot("")) + _, cached := broker.store.nodeInfo("peer", time.Now()) + assert.False(t, cached) +} + +func TestClusterTrustChange_PreservesProofsForUnchangedPins(t *testing.T) { + b := newManualTestBroker() + b.clusterDir = t.TempDir() + clustertrusttest.Join(t, b.clusterDir, "cluster", "self", "peer") + mesh := clustertrust.Open(b.clusterDir) + peer := EnrichedNode{HostUUID: "peer", DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, Trusted: true} + b.store.Upsert(peer, sourceScanner) + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", TelemetryValid: true, MSSince: 137}) + + clustertrusttest.WritePeerPin(t, b.clusterDir, "new-peer") + b.applyClusterTrustChange() + + node, present := b.store.sourceNode("peer", sourceScanner) + assert.True(t, present) + assert.Equal(t, peer, node) + samples := b.telemetry.Snapshot(time.Now()) + require.Len(t, samples, 1) + assert.True(t, samples[0].TelemetryValid) + assert.GreaterOrEqual(t, samples[0].MSSince, int64(137)) +} + +func TestClusterTrustChange_FailsWorkloadsForRemovedPeer(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const workloadID = "running-on-peer" + + test := func(name string, sources ...nodeSource) { + t.Helper() + t.Run(name, func(t *testing.T) { + broker := newManualTestBroker() + broker.nodeID = selfUUID + broker.clusterDir = t.TempDir() + broker.workloads = workloadstore.New() + clustertrusttest.Join(t, broker.clusterDir, "cluster", selfUUID, peerUUID) + mesh := clustertrust.Open(broker.clusterDir) + for _, source := range sources { + switch source { + case sourceScanner: + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, DiscoveryRecordID: mesh.PeerRecordID(peerUUID), + IdentityVerification: noderec.IdentityPinned, Trusted: true, + }, sourceScanner) + case sourceManual: + broker.upsertManualNode(manualNodeStatus{ + ID: "configured", Address: "192.0.2.20", HostUUID: peerUUID, + DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, + }) + } + require.True(t, broker.store.verifiedSource(peerUUID, source)) + } + require.True(t, broker.workloads.Apply(storeIncoming(t, workloadID, selfUUID, "ollama", "run", "running", peerUUID))) + + // Removing trust must retire the peer's jobs after every source has + // withdrawn its record, regardless of which source is removed last. + clustertrusttest.RemovePeerPin(t, broker.clusterDir, peerUUID) + broker.applyClusterTrustChange() + + assert.False(t, broker.store.verifiedSource(peerUUID, sourceScanner)) + assert.False(t, broker.store.verifiedSource(peerUUID, sourceManual)) + workload, present := broker.workloads.Get(selfUUID, workloadID) + require.True(t, present) + assert.Equal(t, "failed", workload.State) + assert.True(t, workload.Inferred) + }) + } + + test("scanner only", sourceScanner) + test("manual only", sourceManual) + test("scanner and manual", sourceScanner, sourceManual) +} + +func TestManualNodes_FailsWorkloadsWhenTrustWithdrawalPrecedesNotification(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const manualID = "configured" + const manualAddress = "192.0.2.20" + const workloadID = "running-on-peer" + + test := func(name string, verification noderec.IdentityVerification) { + t.Helper() + t.Run(name, func(t *testing.T) { + broker := newManualTestBroker() + broker.nodeID = selfUUID + broker.clusterDir = t.TempDir() + broker.workloads = workloadstore.New() + clustertrusttest.Join(t, broker.clusterDir, "cluster", selfUUID, peerUUID) + mesh := clustertrust.Open(broker.clusterDir) + status := manualNodeStatus{ + ID: manualID, Address: manualAddress, HostUUID: peerUUID, + DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, + } + broker.upsertManualNode(status) + require.True(t, broker.store.verifiedSource(peerUUID, sourceManual)) + require.True(t, broker.workloads.Apply(storeIncoming(t, workloadID, selfUUID, "ollama", "run", "running", peerUUID))) + + update := manualNodeStatus{ID: manualID, Address: manualAddress, IdentityVerification: verification} + if verification == noderec.IdentityPinned { + // A queued update still carries the authorization that just expired. + update.HostUUID = peerUUID + update.DiscoveryRecordID = status.DiscoveryRecordID + } + params, err := json.Marshal(update) + require.NoError(t, err) + + // Different workers use independent streams. The manual update may + // erase the peer's last record before the trust notification arrives. + clustertrusttest.RemovePeerPin(t, broker.clusterDir, peerUUID) + broker.forwardManualNodesNotification("node/updated", params) + require.False(t, broker.store.verifiedSource(peerUUID, sourceManual)) + broker.applyClusterTrustChange() + + workload, present := broker.workloads.Get(selfUUID, workloadID) + require.True(t, present) + assert.Equal(t, "failed", workload.State) + assert.True(t, workload.Inferred) + }) + } + + test("worker reports an unverified withdrawal", noderec.IdentityUnverified) + test("broker downgrades a queued verified update", noderec.IdentityPinned) +} + +func TestManualNodes_PreservesWorkloadsAfterUnverifiedUpdateWithCurrentPin(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const manualID = "configured" + const manualAddress = "192.0.2.20" + const workloadID = "running-on-peer" + const runningState = "running" + + broker := newManualTestBroker() + broker.nodeID = selfUUID + broker.clusterDir = t.TempDir() + broker.workloads = workloadstore.New() + clustertrusttest.Join(t, broker.clusterDir, "cluster", selfUUID, peerUUID) + mesh := clustertrust.Open(broker.clusterDir) + broker.upsertManualNode(manualNodeStatus{ + ID: manualID, Address: manualAddress, HostUUID: peerUUID, + DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, + }) + require.True(t, broker.store.verifiedSource(peerUUID, sourceManual)) + require.True(t, broker.workloads.Apply(storeIncoming(t, workloadID, selfUUID, "ollama", "run", runningState, peerUUID))) + params, err := json.Marshal(manualNodeStatus{ + ID: manualID, Address: manualAddress, IdentityVerification: noderec.IdentityUnverified, + }) + require.NoError(t, err) + + // An unsuccessful hardware probe does not revoke the peer's authorization + // or establish that an inference request running on that peer has failed. + broker.forwardManualNodesNotification("node/updated", params) + broker.applyClusterTrustChange() + + assert.False(t, broker.store.verifiedSource(peerUUID, sourceManual)) + workload, present := broker.workloads.Get(selfUUID, workloadID) + require.True(t, present) + assert.Equal(t, runningState, workload.State) + assert.False(t, workload.Inferred) +} + +func TestClusterTrustChange_FailsWorkloadsWhenWorkerCleanupPrecedesNotification(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const workloadID = "running-on-peer" + + test := func(name string, source nodeSource, cleanup func(*Broker)) { + t.Helper() + t.Run(name, func(t *testing.T) { + broker := newManualTestBroker() + broker.nodeID = selfUUID + broker.clusterDir = t.TempDir() + broker.workloads = workloadstore.New() + clustertrusttest.Join(t, broker.clusterDir, "cluster", selfUUID, peerUUID) + mesh := clustertrust.Open(broker.clusterDir) + switch source { + case sourceScanner: + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, DiscoveryRecordID: mesh.PeerRecordID(peerUUID), + IdentityVerification: noderec.IdentityPinned, Trusted: true, + }, sourceScanner) + case sourceManual: + broker.upsertManualNode(manualNodeStatus{ + ID: "configured", Address: "192.0.2.20", HostUUID: peerUUID, + DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, + }) + } + require.True(t, broker.store.verifiedSource(peerUUID, source)) + require.True(t, broker.workloads.Apply(storeIncoming(t, workloadID, selfUUID, "ollama", "run", "running", peerUUID))) + + // The crash discards the directory record before the broker learns + // why the peer's authorization changed. Trust loss still retires jobs. + clustertrusttest.RemovePeerPin(t, broker.clusterDir, peerUUID) + cleanup(broker) + broker.applyClusterTrustChange() + + assert.False(t, broker.store.verifiedSource(peerUUID, source)) + workload, present := broker.workloads.Get(selfUUID, workloadID) + require.True(t, present) + assert.Equal(t, "failed", workload.State) + assert.True(t, workload.Inferred) + }) + } + + test("scanner crashes after pin removal", sourceScanner, (*Broker).clearScannerState) + test("manual worker crashes after pin removal", sourceManual, (*Broker).clearManualNodesState) +} + +func TestClusterTrustChange_PreservesWorkloadsWhileAnotherSourceIsCurrent(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const clusterID = "cluster" + const workloadID = "running-on-peer" + const runningState = "running" + + broker := newManualTestBroker() + broker.nodeID = selfUUID + broker.clusterDir = t.TempDir() + broker.workloads = workloadstore.New() + clustertrusttest.Join(t, broker.clusterDir, clusterID, selfUUID, peerUUID) + mesh := clustertrust.Open(broker.clusterDir) + broker.store.Upsert(EnrichedNode{ + HostUUID: peerUUID, DiscoveryRecordID: mesh.PeerRecordID(peerUUID), + IdentityVerification: noderec.IdentityPinned, Trusted: true, + }, sourceScanner) + require.True(t, broker.workloads.Apply(storeIncoming(t, workloadID, selfUUID, "ollama", "run", runningState, peerUUID))) + + // The manual worker authenticated again before the broker processed the + // trust change. Its current record survives the scanner's obsolete one. + clustertrusttest.WriteAdmission(t, broker.clusterDir, clusterID, 2) + mesh.Refresh() + broker.upsertManualNode(manualNodeStatus{ + ID: "configured", Address: "192.0.2.20", HostUUID: peerUUID, + DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, + }) + + broker.applyClusterTrustChange() + + assert.False(t, broker.store.verifiedSource(peerUUID, sourceScanner)) + assert.True(t, broker.store.verifiedSource(peerUUID, sourceManual)) + workload, present := broker.workloads.Get(selfUUID, workloadID) + require.True(t, present) + assert.Equal(t, runningState, workload.State) + assert.False(t, workload.Inferred) +} + +// A crashed scanner must lose its directory records and measurements, and late +// notifications from that process must not restore them. Independently owned +// manual entries, cached local inventory, and running workloads must survive. +func TestScannerRestart_WithdrawsRecordsAndRejectsOldWorkerEvents(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const workloadID = "running-on-peer" + const runningState = "running" + + b := newManualTestBroker() + b.nodeID = selfUUID + b.clusterDir = t.TempDir() + b.workloads = workloadstore.New() + clustertrusttest.Join(t, b.clusterDir, "cluster", selfUUID, peerUUID) + mesh := clustertrust.Open(b.clusterDir) + b.relayDir = relay.NewDirectory() + scanner := &scannerProcess{store: b.store, relayDir: b.relayDir, stateMu: &b.discoveryMu} + b.setScanner(scanner) + + // Give the scanner a verified peer record and a cached GPU usage sample. + peer := noderec.DirectoryNode{HostUUID: peerUUID, DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, Trusted: true} + params, err := json.Marshal(noderec.NodeEvent{Node: peer}) + require.NoError(t, err) + scanner.handleNotify(noderec.NotifyNodeDiscovered, params) + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: peerUUID, TelemetryValid: true}) + // The peer keeps executing this job even when our local scanner crashes. + require.True(t, b.workloads.Apply(storeIncoming(t, workloadID, selfUUID, "ollama", "run", runningState, peerUUID))) + + // These belong to other workers. Keep an older local receipt time so the + // final assertion also checks that scanner cleanup does not make it look new. + b.upsertManualNode(manualNodeStatus{ID: "manual", Address: "192.0.2.30"}) + owned := localInfoObservation{info: noderec.NodeInfo{HostUUID: selfUUID, TelemetryValid: true}, receivedAt: time.Now().Add(-time.Minute)} + b.localInfo = &owned + + // Run the cleanup used after a scanner crash, then simulate an update that + // arrives late from the retired process and tries to restore its peer record. + b.clearScannerState() + scanner.handleNotify(noderec.NotifyNodeUpdated, params) + + // The old scanner cannot restore its records or scheduling measurements. + assert.Nil(t, b.getScanner()) + assert.Empty(t, b.telemetry.Snapshot(time.Now())) + assert.Empty(t, b.relayDir.Snapshot("")) + + // Manual configuration and the original local inventory observation survive. + snapshot := b.store.Snapshot() + require.Len(t, snapshot, 1) + assert.Equal(t, "manual:manual", snapshot[0].HostUUID) + assert.Equal(t, owned, *b.localInfo) + workload, present := b.workloads.Get(selfUUID, workloadID) + require.True(t, present) + assert.Equal(t, runningState, workload.State) + assert.False(t, workload.Inferred) +} + +func TestManualWorkerRestart_RejectsNotificationsFromPreviousWorker(t *testing.T) { + b := newManualTestBroker() + oldWorker := b.manualNotificationHandler(b.manualGeneration) + status := manualNodeStatus{ID: "manual", Address: "192.0.2.20", OllamaUp: true, OllamaPort: 11434} + params, err := json.Marshal(status) + require.NoError(t, err) + oldWorker("node/updated", params) + require.Len(t, b.store.Snapshot(), 1) + + b.clearManualNodesState() + oldWorker("node/updated", params) + assert.Empty(t, b.store.Snapshot()) + assert.Empty(t, b.manualNodeStatuses) + + b.manualNotificationHandler(b.manualGeneration)("node/updated", params) + assert.Len(t, b.store.Snapshot(), 1) +} + +func TestManualNodes_RejectsQueuedProofAfterPinReplacement(t *testing.T) { + b := newManualTestBroker() + b.clusterDir = t.TempDir() + clustertrusttest.Join(t, b.clusterDir, "cluster", "self", "peer") + mesh := clustertrust.Open(b.clusterDir) + status := manualNodeStatus{ + ID: "configured", HostUUID: "peer", Address: "192.0.2.20", + DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, + OllamaUp: true, OllamaPort: 11434, TelemetryValid: true, + } + send := func() { + t.Helper() + params, err := json.Marshal(status) + require.NoError(t, err) + b.forwardManualNodesNotification("node/updated", params) + } + send() + require.True(t, b.store.verifiedSource("peer", sourceManual)) + clustertrusttest.WritePeerPin(t, b.clusterDir, "peer") + + send() + + assert.False(t, b.store.verifiedSource("peer", sourceManual)) + assert.Empty(t, b.telemetry.Snapshot(time.Now())) + assert.Equal(t, "configured", b.manualAliasForKey("manual:configured")) + mesh.Refresh() + status.DiscoveryRecordID = mesh.PeerRecordID("peer") + send() + assert.True(t, b.store.verifiedSource("peer", sourceManual)) + assert.Len(t, b.telemetry.Snapshot(time.Now()), 1) +} + +// A directory update becomes visible before its new GPU sample is cached. +// Replay can run in that interval, so a replacement record must first discard +// measurements belonging to the previous authorization. +func TestManualNodes_RecordReplacementDiscardsTelemetryBeforeDiscovery(t *testing.T) { + const clusterID = "cluster" + const selfUUID = "self" + const peerUUID = "peer" + const oldUtilization = uint32(80) + const newUtilization = uint32(10) + + test := func(name string, admissionEpoch uint64, wantReplay []uint32) { + t.Run(name, func(t *testing.T) { + broker := newManualTestBroker() + broker.clusterDir = t.TempDir() + clustertrusttest.Join(t, broker.clusterDir, clusterID, selfUUID, peerUUID) + mesh := clustertrust.Open(broker.clusterDir) + status := manualNodeStatus{ + ID: "configured", HostUUID: peerUUID, Address: "192.0.2.20", + DiscoveryRecordID: mesh.PeerRecordID(peerUUID), IdentityVerification: noderec.IdentityPinned, + TelemetryValid: true, GPUs: []GPUInfo{{UtilizationPercent: oldUtilization}}, + } + broker.upsertManualNode(status) + + // A new admission changes the record ID without changing the host UUID. + // Rewriting the same epoch leaves authorization unchanged in the control case. + clustertrusttest.WriteAdmission(t, broker.clusterDir, clusterID, admissionEpoch) + mesh.Refresh() + status.DiscoveryRecordID = mesh.PeerRecordID(peerUUID) + status.GPUs = []GPUInfo{{UtilizationPercent: newUtilization}} + params, err := json.Marshal(status) + require.NoError(t, err) + + // Replay at the directory-change hook reproduces the interval before + // the update reaches telemetry ingestion, without timing-dependent goroutines. + var output bytes.Buffer + scheduler := &rpcWorker{peer: NewPeer(NewCodec(&output))} + var replayed int + broker.store.SetOnChange(func() { + broker.schedulerFeedMu.Lock() + defer broker.schedulerFeedMu.Unlock() + replayed = broker.replayTelemetryToScheduler(scheduler) + }) + + broker.forwardManualNodesNotification("node/updated", params) + + require.Equal(t, len(wantReplay), replayed) + codec := NewCodec(&output) + for _, utilization := range wantReplay { + message, err := codec.Read() + require.NoError(t, err) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, peerUUID, sample.HostUUID) + assert.True(t, sample.TelemetryValid) + assert.Equal(t, utilization, sample.GPUUtilizationPct) + } + _, err = codec.Read() + assert.ErrorIs(t, err, io.EOF) + + // The newly authenticated measurement remains available after the update. + samples := broker.telemetry.Snapshot(time.Now()) + require.Len(t, samples, 1) + assert.True(t, samples[0].TelemetryValid) + assert.Equal(t, newUtilization, samples[0].GPUUtilizationPct) + }) + } + + test("new admission discards the previous measurement", 2, nil) + test("unchanged authorization retains the previous measurement", 1, []uint32{oldUtilization}) +} diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 054652951..932066153 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -15,6 +15,7 @@ import ( "time" "nvpair-shared/applog" + "nvpair-shared/clustertrust" "nvpair-shared/netpick" "nvpair-shared/noderec" @@ -271,6 +272,36 @@ func newDiscoveryStore() *discoveryStore { return &discoveryStore{nodes: make(map[string]storedNode)} } +func (s *discoveryStore) sourceNodes(source nodeSource) []EnrichedNode { + s.mu.RLock() + defer s.mu.RUnlock() + var nodes []EnrichedNode + for _, stored := range s.nodes { + node := stored.scanner + if source == sourceManual { + node = stored.manual + } + if node != nil { + nodes = append(nodes, *node) + } + } + return nodes +} + +func (s *discoveryStore) sourceNode(key string, source nodeSource) (EnrichedNode, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + stored := s.nodes[key] + node := stored.scanner + if source == sourceManual { + node = stored.manual + } + if node == nil { + return EnrichedNode{}, false + } + return *node, true +} + // SetOnChange installs (or removes, with nil) the post-mutation hook. // Setting it after a sequence of mutations is the broker's way of // silencing the notification stream during startup: by attaching the @@ -295,6 +326,9 @@ func (s *discoveryStore) Upsert(n EnrichedNode, source nodeSource) { cp := n s.mu.Lock() sn := s.nodes[key] + if source == sourceScanner && sn.scanner != nil && sn.scanner.DiscoveryRecordID != n.DiscoveryRecordID { + sn.info = nil + } sn.setSource(source, &cp) sn.lastSeen = time.Now() s.nodes[key] = sn @@ -434,6 +468,9 @@ func (sn storedNode) toAvailable() AvailableNode { // subscribe to the promoted daemon and reading its acks — while // the scanner's node/* notifications are routed into the discovery store. type scannerProcess struct { + stateMu *sync.Mutex + mesh *clustertrust.Mesh + stopped bool // protected by stateMu, alongside directory mutations cmd *exec.Cmd stdin io.WriteCloser peer *Peer @@ -494,6 +531,8 @@ func (s *scannerProcess) Done() <-chan struct{} { return s.done } func startScanner( binaryPath, logLevel string, store *discoveryStore, + stateMu *sync.Mutex, + mesh *clustertrust.Mesh, relayDir *relay.Directory, onNodeLost func(uuid, name string), onTelemetry func(noderec.NodeTelemetry), @@ -522,6 +561,8 @@ func startScanner( } sp := &scannerProcess{ + stateMu: stateMu, + mesh: mesh, cmd: cmd, stdin: stdin, peer: NewPeer(NewCodec(readWriter{stdout, stdin})), @@ -691,6 +732,17 @@ func (s *scannerProcess) pushUnregister(svc noderec.ServiceKey) { // is logged. Responses to broker-issued Calls are demuxed by the peer itself and // never reach here; any other notification is ignored. func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { + receivedAt := time.Now() + if s.stateMu != nil { + s.stateMu.Lock() + defer s.stateMu.Unlock() + } + if s.stopped { + return + } + if s.mesh != nil { + s.mesh.Refresh() + } switch method { case noderec.NotifyNodeDiscovered, noderec.NotifyNodeUpdated, noderec.NotifyNodeRemoved: var ev noderec.NodeEvent @@ -698,6 +750,16 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { slog.Warn("daemon emitted invalid node event", "method", method, "err", err) return } + if method != noderec.NotifyNodeRemoved && !s.currentRecord(ev.Node.HostUUID, ev.Node.DiscoveryRecordID, ev.Node.IdentityVerification) { + return + } + if method == noderec.NotifyNodeRemoved && s.store != nil { + // A delayed removal for a previous pin must not remove its newly + // authenticated replacement under the same canonical host UUID. + if node, present := s.store.sourceNode(directoryToEnriched(ev.Node).storeKey(), sourceScanner); present && node.DiscoveryRecordID != ev.Node.DiscoveryRecordID { + return + } + } if method != noderec.NotifyNodeRemoved && s.onLocalNode != nil { s.onLocalNode(ev.Node) } @@ -726,6 +788,10 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { s.onNodeLost(ev.Node.HostUUID, ev.Node.Name) } } else { + previous, present := s.store.sourceNode(directoryToEnriched(ev.Node).storeKey(), sourceScanner) + if present && previous.DiscoveryRecordID != ev.Node.DiscoveryRecordID && s.onTelemetryRemoved != nil { + s.onTelemetryRemoved(ev.Node.HostUUID) + } s.store.Upsert(directoryToEnriched(ev.Node), sourceScanner) } } @@ -739,12 +805,20 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { slog.Warn("daemon emitted node telemetry without hostUuid") return } + if !s.currentHost(telemetry.HostUUID) { + return + } if s.store != nil && s.store.verifiedSource(telemetry.HostUUID, sourceScanner) && s.onTelemetry != nil { + telemetry = telemetry.Aged(time.Since(receivedAt)) s.onTelemetry(telemetry) } case noderec.NotifyNodeInfo: var value noderec.NodeInfoEvent - if json.Unmarshal(params, &value) != nil || s.store == nil || !s.store.AcceptInfo(value, time.Now()) { + if json.Unmarshal(params, &value) != nil || s.store == nil || !s.currentRecord(value.HostUUID, value.DiscoveryRecordID, value.IdentityVerification) { + return + } + value.Info = value.Info.Aged(time.Since(receivedAt)) + if !s.store.AcceptInfo(value, time.Now()) { return } if s.onNodeInfo != nil { @@ -755,6 +829,21 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { } } +func (s *scannerProcess) currentRecord(uuid, recordID string, verification noderec.IdentityVerification) bool { + if s.mesh == nil || verification != noderec.IdentityPinned { + return true + } + return recordID != "" && recordID == s.mesh.PeerRecordID(uuid) +} + +func (s *scannerProcess) currentHost(uuid string) bool { + if s.store == nil { + return false + } + node, present := s.store.sourceNode(uuid, sourceScanner) + return present && node.IdentityVerification != noderec.IdentityUnverified && s.currentRecord(uuid, node.DiscoveryRecordID, node.IdentityVerification) +} + // writeSetLevelFrame marshals a newline-delimited log/set-level // notification and writes it to a child's stdin under mu. The level is // assumed already validated by the caller (applog.HandleSetLevelParams in diff --git a/services/nvpair-ui-broker/discovery_test.go b/services/nvpair-ui-broker/discovery_test.go index 70513bbfa..b247126db 100644 --- a/services/nvpair-ui-broker/discovery_test.go +++ b/services/nvpair-ui-broker/discovery_test.go @@ -5,15 +5,98 @@ package main import ( "encoding/json" + "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" ) +func TestScannerTelemetry_IncludesDiscoveryLockWaitInSampleAge(t *testing.T) { + store := newDiscoveryStore() + store.Upsert(EnrichedNode{HostUUID: "self", IdentityVerification: noderec.IdentityLocal}, sourceScanner) + var stateMu sync.Mutex + var received noderec.NodeTelemetry + scanner := &scannerProcess{ + store: store, stateMu: &stateMu, + onTelemetry: func(sample noderec.NodeTelemetry) { received = sample }, + } + wait := holdDiscoveryNotification(t, &stateMu, func() { + scanner.handleNotify(noderec.NotifyNodeTelemetry, json.RawMessage(`{"hostUuid":"self","telemetryValid":true,"msSince":137}`)) + }) + + assert.True(t, received.TelemetryValid) + assert.GreaterOrEqual(t, received.MSSince, int64(137)+wait.Milliseconds()) +} + +func TestScannerNodeInfo_IncludesDiscoveryLockWaitInSampleAge(t *testing.T) { + store := newDiscoveryStore() + store.Upsert(EnrichedNode{HostUUID: "self", DiscoveryRecordID: "local:self", IdentityVerification: noderec.IdentityLocal}, sourceScanner) + var stateMu sync.Mutex + scanner := &scannerProcess{store: store, stateMu: &stateMu} + wait := holdDiscoveryNotification(t, &stateMu, func() { + scanner.handleNotify(noderec.NotifyNodeInfo, json.RawMessage(`{"hostUuid":"self","discoveryRecordId":"local:self","identityVerification":"local","info":{"hostUuid":"self","telemetryValid":true,"msSince":137}}`)) + }) + + value, present := store.nodeInfo("self", time.Now()) + require.True(t, present) + assert.GreaterOrEqual(t, value.Info.MSSince, int64(137)+wait.Milliseconds()) +} + +func TestDiscoveryStore_RecordReplacementDiscardsHardwareSnapshot(t *testing.T) { + store := newDiscoveryStore() + node := EnrichedNode{HostUUID: "peer", DiscoveryRecordID: "old-proof", IdentityVerification: noderec.IdentityPinned, Trusted: true} + store.Upsert(node, sourceScanner) + principal := "peer" + require.True(t, store.AcceptInfo(noderec.NodeInfoEvent{ + HostUUID: "peer", ClusterUUID: principal, DiscoveryRecordID: "old-proof", + IdentityVerification: noderec.IdentityPinned, + Info: noderec.NodeInfo{HostUUID: "peer", ClusterUUID: &principal}, + }, time.Now())) + + node.DiscoveryRecordID = "new-proof" + store.Upsert(node, sourceScanner) + + _, cached := store.nodeInfo("peer", time.Now()) + assert.False(t, cached) +} + +func TestScannerProcess_RejectsQueuedEventsAfterPinReplacement(t *testing.T) { + dir := t.TempDir() + clustertrusttest.Join(t, dir, "cluster", "self", "peer") + mesh := clustertrust.Open(dir) + store := newDiscoveryStore() + scanner := &scannerProcess{store: store, mesh: mesh} + node := noderec.DirectoryNode{HostUUID: "peer", DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, Trusted: true} + send := func(method string, node noderec.DirectoryNode) { + t.Helper() + params, err := json.Marshal(noderec.NodeEvent{Node: node}) + require.NoError(t, err) + scanner.handleNotify(method, params) + } + send(noderec.NotifyNodeDiscovered, node) + require.True(t, store.verifiedSource("peer", sourceScanner)) + store.Remove("peer", sourceScanner) + clustertrusttest.WritePeerPin(t, dir, "peer") + + send(noderec.NotifyNodeUpdated, node) + assert.Empty(t, store.Snapshot()) + mesh.Refresh() + replacement := node + replacement.DiscoveryRecordID = mesh.PeerRecordID("peer") + send(noderec.NotifyNodeDiscovered, replacement) + send(noderec.NotifyNodeRemoved, node) + + snapshot := store.Snapshot() + require.Len(t, snapshot, 1) + assert.Equal(t, replacement.DiscoveryRecordID, snapshot[0].DiscoveryRecordID) +} + // Store a verified peer alongside an unverified advertisement copying its UUID // and name. Removing the advertisement must leave the peer's record, endpoint, // and trust intact, rather than overwriting or removing that peer. @@ -508,7 +591,7 @@ func TestScannerProcess_RoutesAndRemovesTelemetry(t *testing.T) { sp.handleNotify(noderec.NotifyNodeTelemetry, params) require.Equal(t, want.HostUUID, got.HostUUID, "routed telemetry (%v, %v)", got, want) require.True(t, got.TelemetryValid, "routed telemetry (%v, %v)", got, want) - require.Equal(t, want.MSSince, got.MSSince, "routed telemetry (%v, %v)", got, want) + require.GreaterOrEqual(t, got.MSSince, want.MSSince, "routed telemetry age includes broker processing") require.Equal(t, uint32(84), got.GPUUtilizationPct, "routed utilization") sp.handleNotify(noderec.NotifyNodeRemoved, nodeRemovedFrame(t, "peer", hostUUID)) diff --git a/services/nvpair-ui-broker/engineproxy.go b/services/nvpair-ui-broker/engineproxy.go index 43a5ff91b..230ca62f2 100644 --- a/services/nvpair-ui-broker/engineproxy.go +++ b/services/nvpair-ui-broker/engineproxy.go @@ -96,6 +96,8 @@ type engineProxyRuntime struct { // supervisor swaps it on restart from the monitor goroutine while request // handlers read it from the read loop. proxy *proxyProcess + // manualBridge owns this facade's ordered manual-target sender. + manualBridge manualProxyBridge // subscribed reports whether a client asked for this engine's relay // stream. Guarded by Broker.proxyMu. diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index 9b70497e4..3bfb4a141 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -10,7 +10,9 @@ import ( "log/slog" "os" "path/filepath" + "slices" "sort" + "sync" "time" "nvpair-shared/appdir" @@ -26,6 +28,7 @@ import ( // (bridgeManualNode); the rest project into the discovery store via // manualToEnriched. type manualNodeStatus struct { + DiscoveryRecordID string `json:"discoveryRecordId,omitempty"` IdentityVerification noderec.IdentityVerification `json:"identityVerification"` ID string `json:"id"` Address string `json:"address"` @@ -48,6 +51,16 @@ type manualNodeStatus struct { HostUUID string `json:"hostUuid,omitempty"` } +// withoutVerification keeps only the explicitly configured engine target. All +// identity-bound inventory and samples are discarded when its proof expires. +func (s manualNodeStatus) withoutVerification() manualNodeStatus { + s.IdentityVerification = noderec.IdentityUnverified + s.DiscoveryRecordID, s.HostUUID = "", "" + s.GPUs, s.CPU, s.Memory = nil, nil, nil + s.TelemetryValid, s.MSSince = false, 0 + return s +} + type manualNodeStatusEntry struct { status manualNodeStatus receivedAt time.Time @@ -251,6 +264,7 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { ModelsByEngine: manualModelsByEngine(s), } if s.IdentityVerification == noderec.IdentityPinned { + en.DiscoveryRecordID = s.DiscoveryRecordID en.GPUs, en.CPU, en.Memory = s.GPUs, s.CPU, s.Memory en.Trusted = true en.Clustered = true @@ -314,81 +328,202 @@ type proxyManualNode struct { Models []string `json:"models,omitempty"` } +type manualProxyTarget struct { + node *proxyManualNode // nil withdraws a previously attempted target + revision uint64 + attempted uint64 +} + +// manualProxyBridge retains current targets rather than an event queue. One +// sender per engine reconciles them outside discoveryMu, so bridge I/O cannot +// hold discovery processing or accumulate a goroutine for every update. Attempted +// revisions include in-flight writes: a later withdrawal must remove a target +// even if its add was applied without an answer. Each status update advances its +// revision, allowing a failed request to be retried without a busy loop. +type manualProxyBridge struct { + mu sync.Mutex + targets map[string]manualProxyTarget + revision uint64 + proxy *proxyProcess + sending bool +} + // bridgeManualNode keeps every supervised proxy's manual-node set in step with // a manual node's per-engine reachability: a node whose Ollama is up is bridged // into ollama-proxy and one whose LM Studio is up into lmstudio-proxy // (idempotent — each proxy upserts on a repeat), while an engine that is not -// (or no longer) reachable is removed from its proxy. Each leg is a no-op when -// that proxy isn't supervised — the bridge only applies when the broker owns -// both ends. +// (or no longer) reachable is removed from its proxy. Desired targets are kept +// while a proxy is unavailable and replayed after its replacement starts. // // Manual nodes are, by definition, the nodes that never appear in the discovery // relay's snapshots — they advertise no _nvpair-node record for the scanner // daemon to carry — so without this explicit add the proxies can't route // inference to them even though both workers are broker-owned. func (b *Broker) bridgeManualNode(s manualNodeStatus, key string) { - b.bridgeToProxy(b.getProxy(), "ollama", s, key, s.OllamaUp, s.OllamaPort, s.OllamaModels) - b.bridgeToProxy(b.getLMStudioProxy(), "lmstudio", s, key, s.LMStudioUp, s.LMStudioPort, s.LMStudioModels) + b.bridgeToProxy(ollamaProxyProfile, s, key, s.OllamaUp, s.OllamaPort, s.OllamaModels) + b.bridgeToProxy(lmstudioProxyProfile, s, key, s.LMStudioUp, s.LMStudioPort, s.LMStudioModels) } -// bridgeToProxy adds the node to p when its engine is reachable, or removes it +// bridgeToProxy requests the node's addition when its engine is reachable, or removal // otherwise. up/port/models are the engine-specific fields the caller pulled // off the node's status. The proxy candidate is keyed by `key` — the node's // operational identity (its hostUuid once node-info reports it, else the manual // id) — the same key the discovery store and scheduler use, so the scheduler's // priority list and scheduledOn resolve to this candidate. -func (b *Broker) bridgeToProxy(p *proxyProcess, engine string, s manualNodeStatus, key string, up bool, port int, models []string) { - if p == nil { - return - } +func (b *Broker) bridgeToProxy(profile engineProxyProfile, s manualNodeStatus, key string, up bool, port int, models []string) { if up && s.Address != "" && port > 0 { node := proxyManualNode{ ID: key, Host: s.Address, Port: port, Addresses: []string{s.Address}, - Models: models, + Models: slices.Clone(models), } - b.callProxyManual(p, engine, "node/add-manual", node, key) + b.queueManualProxyTarget(profile, key, &node) return } // Engine unreachable (down, or this node doesn't run it): make sure the // proxy isn't left holding a stale manual entry it would try to route to. - b.callProxyManual(p, engine, "node/remove-manual", map[string]string{"id": key}, key) + b.queueManualProxyTarget(profile, key, nil) } // removeManualNodeFromProxies drops a manual node from every supervised proxy. // Idempotent: a no-op for a proxy where the node was never bridged or that // isn't supervised (the proxy's RemoveManual just reports removed=false). func (b *Broker) removeManualNodeFromProxies(id string) { - b.callProxyManual(b.getProxy(), "ollama", "node/remove-manual", map[string]string{"id": id}, id) - b.callProxyManual(b.getLMStudioProxy(), "lmstudio", "node/remove-manual", map[string]string{"id": id}, id) + b.queueManualProxyTarget(ollamaProxyProfile, id, nil) + b.queueManualProxyTarget(lmstudioProxyProfile, id, nil) +} + +func (b *Broker) queueManualProxyTarget(profile engineProxyProfile, key string, node *proxyManualNode) { + bridge := &b.engineProxy(profile).manualBridge + bridge.mu.Lock() + defer bridge.mu.Unlock() + if bridge.targets == nil { + bridge.targets = make(map[string]manualProxyTarget) + } + target := bridge.targets[key] + if node == nil && target.attempted == 0 { + // An add canceled before any send cannot have left a route behind. + delete(bridge.targets, key) + return + } + bridge.revision++ + target.node, target.revision = node, bridge.revision + bridge.targets[key] = target + if !bridge.sending { + bridge.sending = true + go b.sendManualProxyTargets(profile) + } +} + +// nextTarget runs under mu. Remove retired keys before adding replacements, +// including when an alias acquires or loses its authenticated host identity. +func (bridge *manualProxyBridge) nextTarget() (string, manualProxyTarget) { + key := "" + for candidate, target := range bridge.targets { + if target.attempted != target.revision { + key = candidate + if target.node == nil { + break + } + } + } + target := bridge.targets[key] + if key != "" { + target.attempted = target.revision + bridge.targets[key] = target + } + return key, target +} + +func (b *Broker) sendManualProxyTargets(profile engineProxyProfile) { + bridge := &b.engineProxy(profile).manualBridge + ctx := b.getServeCtx() + for { + bridge.mu.Lock() + proxy := b.engineProxyHandle(profile) + if bridge.proxy != proxy { + bridge.proxy = proxy + for key, target := range bridge.targets { + if target.node == nil { + delete(bridge.targets, key) // a new process starts with no old routes + } else { + target.attempted = 0 + bridge.targets[key] = target + } + } + } + if proxy == nil || (ctx != nil && ctx.Err() != nil) { + bridge.sending = false + bridge.mu.Unlock() + return + } + key, target := bridge.nextTarget() + if key == "" { + bridge.sending = false + bridge.mu.Unlock() + return + } + bridge.mu.Unlock() + if target.node == nil { + if b.callProxyManual(proxy, profile.Name, "node/remove-manual", map[string]string{"id": key}, key) { + bridge.mu.Lock() + if bridge.targets[key].revision == target.revision { + delete(bridge.targets, key) + } + bridge.mu.Unlock() + } + } else { + b.callProxyManual(proxy, profile.Name, "node/add-manual", target.node, key) + } + } +} + +// Facades are enabled after their handles are published. Force a replay after +// startup so any add rejected during that interval is retried on the live facade. +func (b *Broker) replayManualProxyTargets() { + for _, profile := range []engineProxyProfile{ollamaProxyProfile, lmstudioProxyProfile} { + bridge := &b.engineProxy(profile).manualBridge + bridge.mu.Lock() + for key, target := range bridge.targets { + bridge.revision++ + target.revision = bridge.revision + bridge.targets[key] = target + } + if !bridge.sending { + bridge.sending = true + go b.sendManualProxyTargets(profile) + } + bridge.mu.Unlock() + } } // callProxyManual issues a best-effort node/add-manual|remove-manual to one // engine's facade. The bridge is advisory plumbing the broker owns, not part of // the manual-node request's own response, so a failure is logged and swallowed // rather than surfaced to the client. A nil proxy (not supervised) is a no-op. -// Runs synchronously on the manual-nodes reader goroutine; a proxy answers -// these control-plane calls locally in well under proxyCallTimeout, and -// manual-node events are infrequent (one per 10s probe), so the brief inline -// call keeps add/remove strictly ordered without a queue. +// Manual discovery updates call this only from their per-engine sender, outside +// discoveryMu; the synchronous exchange keeps each engine's writes ordered. // // Every method routed here is facade-scoped — a manual node belongs to one // engine's routing set — so the engine is addressed on the wire rather than // left implicit in which process was picked. -func (b *Broker) callProxyManual(p *proxyProcess, engine, method string, params any, id string) { +func (b *Broker) callProxyManual(p *proxyProcess, engine, method string, params any, id string) bool { if p == nil { - return + return false } raw, err := json.Marshal(params) if err != nil { slog.Warn("manual->proxy bridge: marshal failed", "engine", engine, "method", method, "id", id, "err", err) - return + return false } if _, rpcErr, err := p.Call(context.Background(), engines.AddressMethod(engine, method), raw); err != nil { slog.Warn("manual->proxy bridge failed", "engine", engine, "method", method, "id", id, "err", err) + return false } else if rpcErr != nil { slog.Warn("manual->proxy bridge rejected", "engine", engine, "method", method, "id", id, "code", rpcErr.Code, "msg", rpcErr.Message) + return false } + return true } diff --git a/services/nvpair-ui-broker/manualnodes_test.go b/services/nvpair-ui-broker/manualnodes_test.go index 4fe036f2b..3d2ce4fde 100644 --- a/services/nvpair-ui-broker/manualnodes_test.go +++ b/services/nvpair-ui-broker/manualnodes_test.go @@ -5,9 +5,12 @@ package main import ( "bytes" + "context" "encoding/json" + "fmt" "net" "path/filepath" + "sync" "testing" "time" @@ -223,3 +226,322 @@ func TestManualToEnriched_ExcludesUnverifiedHardwareFromServiceRecords(t *testin assert.Equal(t, "manual:configured", node.HostUUID) assert.Equal(t, []string{modelName}, node.Models) } + +type observedProxyWriter struct { + net.Conn + writing chan struct{} + once sync.Once +} + +func (w *observedProxyWriter) Write(data []byte) (int, error) { + w.once.Do(func() { close(w.writing) }) + return w.Conn.Write(data) +} + +// A proxy that stops reading or answering must not hold up local hardware +// updates or prevent the broker from accepting the manual target's removal. +func TestManualProxyBridge_DoesNotBlockDiscovery(t *testing.T) { + test := func(name string, blockWrite bool) { + t.Run(name, func(t *testing.T) { + brokerSide, proxySide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, brokerSide.Close()) + assert.NoError(t, proxySide.Close()) + }) + require.NoError(t, proxySide.SetDeadline(time.Now().Add(5*time.Second))) + writer := &observedProxyWriter{Conn: brokerSide, writing: make(chan struct{})} + proxy := &proxyProcess{peer: NewPeer(NewCodec(writer))} + go proxy.peer.Serve(nil, nil) + b := newManualTestBroker() + b.nodeID = "self" + b.setProxy(proxy) + codec := NewCodec(proxySide) + added := make(chan struct{}) + go func() { + b.forwardManualNodesNotification("node/discovered", json.RawMessage(`{"id":"lab","address":"192.0.2.20","ollama_up":true,"ollama_port":11434}`)) + close(added) + }() + select { + case <-writer.writing: + case <-time.After(2 * time.Second): + require.FailNow(t, "manual target was not sent to the proxy") + } + var first *Message + if !blockWrite { + var err error + first, err = codec.Read() + require.NoError(t, err) + } + + owned := make(chan struct{}) + go func() { + b.nodeInfoNotificationHandler(0)(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"self"}}`)) + close(owned) + }() + removed := make(chan struct{}) + go func() { + b.forwardManualNodesNotification("node/removed", json.RawMessage(`{"id":"lab"}`)) + close(removed) + }() + finished := func(done <-chan struct{}, operation string) { + t.Helper() + select { + case <-done: + case <-time.After(time.Second): + assert.Fail(t, operation+" waited for the stalled proxy") + } + } + finished(owned, "owned hardware update") + finished(removed, "manual target removal") + + if first == nil { + var err error + first, err = codec.Read() + require.NoError(t, err) + } + require.Equal(t, "ollama:node/add-manual", first.Method) + require.NoError(t, codec.Respond(first.ID, map[string]bool{"added": true})) + withdrawal, err := codec.Read() + require.NoError(t, err) + assert.Equal(t, "ollama:node/remove-manual", withdrawal.Method) + var params struct { + ID string `json:"id"` + } + require.NoError(t, json.Unmarshal(withdrawal.Params, ¶ms)) + assert.Equal(t, "manual:lab", params.ID) + require.NoError(t, codec.Respond(withdrawal.ID, map[string]bool{"removed": true})) + finished(added, "initial manual update") + finished(removed, "manual target removal after proxy recovery") + finished(owned, "owned hardware update after proxy recovery") + waitForManualProxySender(t, b) + }) + } + test("delayed reply", false) + test("blocked pipe write", true) +} + +func manualProxyTestPeer(t *testing.T, b *Broker) (*Codec, net.Conn) { + t.Helper() + brokerSide, proxySide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, brokerSide.Close()) + assert.NoError(t, proxySide.Close()) + }) + require.NoError(t, proxySide.SetDeadline(time.Now().Add(3*time.Second))) + proxy := &proxyProcess{peer: NewPeer(NewCodec(brokerSide))} + go proxy.peer.Serve(nil, nil) + b.setProxy(proxy) + return NewCodec(proxySide), brokerSide +} + +func waitForManualProxySender(t *testing.T, b *Broker) { + t.Helper() + bridge := &b.ollamaState().manualBridge + // Joining the sender also proves that it has no unexpected extra request + // waiting for a reply, without asserting an exact goroutine schedule. + require.Eventually(t, func() bool { + bridge.mu.Lock() + defer bridge.mu.Unlock() + return !bridge.sending + }, time.Second, time.Millisecond) +} + +// While one add waits for its reply, intermediate probes and aliases can change. +// Recovery must withdraw the retired key before adding only the latest target. +func TestManualProxyBridge_WithdrawsOldKeyBeforeSendingLatestTarget(t *testing.T) { + b := newManualTestBroker() + codec, _ := manualProxyTestPeer(t, b) + first := manualStatus("first", "192.0.2.20", "old-peer") + first.OllamaUp, first.OllamaPort = true, 11434 + b.upsertManualNode(first) + initial, err := codec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:node/add-manual", initial.Method) + second := first + second.ID, second.Address = "second", "192.0.2.21" + b.upsertManualNode(second) + for i := range 20 { + b.upsertManualNode(first) + // Configurations canceled before a send must not accumulate removals. + transient := manualNodeStatus{ID: fmt.Sprintf("transient-%d", i), Address: "192.0.2.40", OllamaUp: true, OllamaPort: 11434} + b.upsertManualNode(transient) + b.removeManualNode(transient.ID) + } + first.HostUUID, first.Address = "new-peer", "192.0.2.30" + first.OllamaModels = []string{"latest-model"} + b.upsertManualNode(first) + b.removeManualNode(second.ID) + require.NoError(t, codec.Respond(initial.ID, map[string]bool{"added": true})) + + withdrawal, err := codec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:node/remove-manual", withdrawal.Method) + var removed struct { + ID string `json:"id"` + } + require.NoError(t, json.Unmarshal(withdrawal.Params, &removed)) + assert.Equal(t, "old-peer", removed.ID) + require.NoError(t, codec.Respond(withdrawal.ID, map[string]bool{"removed": true})) + addition, err := codec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:node/add-manual", addition.Method) + var target proxyManualNode + require.NoError(t, json.Unmarshal(addition.Params, &target)) + assert.Equal(t, "new-peer", target.ID) + assert.Equal(t, "192.0.2.30", target.Host) + assert.Equal(t, []string{"latest-model"}, target.Models) + require.NoError(t, codec.Respond(addition.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) +} + +// The removed alias may have supplied the projected address. Its surviving +// alias must replace that address even when an older add is still in flight. +func TestManualProxyBridge_UsesSurvivingAliasAfterRemoval(t *testing.T) { + b := newManualTestBroker() + codec, _ := manualProxyTestPeer(t, b) + first := manualStatus("first", "192.0.2.20", "peer") + first.OllamaUp, first.OllamaPort = true, 11434 + b.upsertManualNode(first) + initial, err := codec.Read() + require.NoError(t, err) + second := first + second.ID, second.Address = "second", "192.0.2.21" + b.upsertManualNode(second) + b.removeManualNode(first.ID) + require.NoError(t, codec.Respond(initial.ID, map[string]bool{"added": true})) + + addition, err := codec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:node/add-manual", addition.Method) + var target proxyManualNode + require.NoError(t, json.Unmarshal(addition.Params, &target)) + assert.Equal(t, "peer", target.ID) + assert.Equal(t, "192.0.2.21", target.Host) + require.NoError(t, codec.Respond(addition.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) +} + +// A replacement proxy starts empty. It must receive the current configuration, +// including changes accepted while the retired proxy was not answering. +func TestManualProxyBridge_ReplaysLatestTargetAfterProxyRestart(t *testing.T) { + b := newManualTestBroker() + oldCodec, oldConnection := manualProxyTestPeer(t, b) + status := manualNodeStatus{ID: "lab", Address: "192.0.2.20", OllamaUp: true, OllamaPort: 11434} + b.upsertManualNode(status) + _, err := oldCodec.Read() + require.NoError(t, err) + status.Address = "192.0.2.30" + b.upsertManualNode(status) + newCodec, _ := manualProxyTestPeer(t, b) + b.replayProxyStateAfterSpawn() + require.NoError(t, oldConnection.Close()) + + addition, err := newCodec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:node/add-manual", addition.Method) + var target proxyManualNode + require.NoError(t, json.Unmarshal(addition.Params, &target)) + assert.Equal(t, "manual:lab", target.ID) + assert.Equal(t, "192.0.2.30", target.Host) + require.NoError(t, newCodec.Respond(addition.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) +} + +// A rejected bridge request is retried when the next probe reports the same +// reachable engine; remembering an attempted snapshot must not suppress it. +func TestManualProxyBridge_RetriesIdenticalProbeAfterFailure(t *testing.T) { + b := newManualTestBroker() + codec, _ := manualProxyTestPeer(t, b) + status := manualNodeStatus{ID: "lab", Address: "192.0.2.20", OllamaUp: true, OllamaPort: 11434} + b.upsertManualNode(status) + first, err := codec.Read() + require.NoError(t, err) + require.NoError(t, codec.RespondError(first.ID, -32000, "facade not enabled")) + waitForManualProxySender(t, b) + + b.upsertManualNode(status) + retried, err := codec.Read() + require.NoError(t, err) + assert.Equal(t, "ollama:node/add-manual", retried.Method) + var target proxyManualNode + require.NoError(t, json.Unmarshal(retried.Params, &target)) + assert.Equal(t, proxyManualNode{ID: "manual:lab", Host: "192.0.2.20", Port: 11434, Addresses: []string{"192.0.2.20"}}, target) + require.NoError(t, codec.Respond(retried.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) +} + +// If a live proxy rejects a removal, later unreachable probes must retry it; +// otherwise the old route survives even though the broker withdrew the target. +func TestManualProxyBridge_RetriesWithdrawalAfterFailure(t *testing.T) { + b := newManualTestBroker() + codec, _ := manualProxyTestPeer(t, b) + status := manualNodeStatus{ID: "lab", Address: "192.0.2.20", OllamaUp: true, OllamaPort: 11434} + b.upsertManualNode(status) + addition, err := codec.Read() + require.NoError(t, err) + require.NoError(t, codec.Respond(addition.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) + status.OllamaUp = false + b.upsertManualNode(status) + withdrawal, err := codec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:node/remove-manual", withdrawal.Method) + require.NoError(t, codec.RespondError(withdrawal.ID, -32000, "temporary failure")) + waitForManualProxySender(t, b) + + b.upsertManualNode(status) + retried, err := codec.Read() + require.NoError(t, err, "the next unreachable probe must retry withdrawal") + assert.Equal(t, "ollama:node/remove-manual", retried.Method) + var params struct { + ID string `json:"id"` + } + require.NoError(t, json.Unmarshal(retried.Params, ¶ms)) + assert.Equal(t, "manual:lab", params.ID) + require.NoError(t, codec.Respond(retried.ID, map[string]bool{"removed": true})) + waitForManualProxySender(t, b) +} + +// No new probe is needed to repopulate an idle target after proxy restart: the +// spawn replay restores the last accepted configuration into the empty process. +func TestManualProxyBridge_ReplaysIdleTargetsAfterProxyRestart(t *testing.T) { + b := newManualTestBroker() + oldCodec, oldConnection := manualProxyTestPeer(t, b) + b.upsertManualNode(manualNodeStatus{ID: "lab", Address: "192.0.2.20", OllamaUp: true, OllamaPort: 11434}) + initial, err := oldCodec.Read() + require.NoError(t, err) + require.NoError(t, oldCodec.Respond(initial.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) + require.NoError(t, oldConnection.Close()) + newCodec, _ := manualProxyTestPeer(t, b) + + b.replayProxyStateAfterSpawn() + addition, err := newCodec.Read() + require.NoError(t, err) + assert.Equal(t, "ollama:node/add-manual", addition.Method) + var target proxyManualNode + require.NoError(t, json.Unmarshal(addition.Params, &target)) + assert.Equal(t, proxyManualNode{ID: "manual:lab", Host: "192.0.2.20", Port: 11434, Addresses: []string{"192.0.2.20"}}, target) + require.NoError(t, newCodec.Respond(addition.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) +} + +// Teardown closes the child pipe. Once its in-flight call completes, a queued +// newer target must not start another exchange with a proxy being shut down. +func TestManualProxyBridge_StopsSendingWhenBrokerShutsDown(t *testing.T) { + b := newManualTestBroker() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + b.setServeCtx(ctx) + codec, _ := manualProxyTestPeer(t, b) + status := manualNodeStatus{ID: "lab", Address: "192.0.2.20", OllamaUp: true, OllamaPort: 11434} + b.upsertManualNode(status) + initial, err := codec.Read() + require.NoError(t, err) + status.Address = "192.0.2.30" + b.upsertManualNode(status) + + cancel() + require.NoError(t, codec.Respond(initial.ID, map[string]bool{"added": true})) + waitForManualProxySender(t, b) +} diff --git a/services/nvpair-ui-broker/nodeinfo.go b/services/nvpair-ui-broker/nodeinfo.go index 629a65e64..eb083954a 100644 --- a/services/nvpair-ui-broker/nodeinfo.go +++ b/services/nvpair-ui-broker/nodeinfo.go @@ -57,8 +57,7 @@ type localInfoObservation struct { // acceptLocalInfo caches an owned-worker snapshot matching this host and queues // it for a running scanner. Receipt time is retained for age-preserving replay. -func (b *Broker) acceptLocalInfo(params json.RawMessage) { - receivedAt := time.Now() +func (b *Broker) acceptLocalInfo(params json.RawMessage, receivedAt time.Time) { var snapshot noderec.LocalNodeInfoParams if err := json.Unmarshal(params, &snapshot); err != nil { slog.Warn("bad owned node-info snapshot", "err", err) diff --git a/services/nvpair-ui-broker/nodeinfo_observed_test.go b/services/nvpair-ui-broker/nodeinfo_observed_test.go index 36eb35c32..da740231a 100644 --- a/services/nvpair-ui-broker/nodeinfo_observed_test.go +++ b/services/nvpair-ui-broker/nodeinfo_observed_test.go @@ -46,7 +46,7 @@ func TestForwardNodeInfoObservedAddresses_ReachesTheScanner(t *testing.T) { relayed <- msg }() - b.forwardNodeInfoNotification(noderec.NotifyObservedAddresses, json.RawMessage(from)) + b.nodeInfoNotificationHandler(0)(noderec.NotifyObservedAddresses, json.RawMessage(from)) select { case msg := <-relayed: diff --git a/services/nvpair-ui-broker/nodeinfo_test.go b/services/nvpair-ui-broker/nodeinfo_test.go index 0acadb2f0..2116b97e5 100644 --- a/services/nvpair-ui-broker/nodeinfo_test.go +++ b/services/nvpair-ui-broker/nodeinfo_test.go @@ -24,7 +24,7 @@ func TestLocalNodeInfo_RelaysOwnedWorkerSamples(t *testing.T) { b := &Broker{nodeID: "owned-host"} sp, messages := localInfoScanner(t) b.setScanner(sp) - b.forwardNodeInfoNotification(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","GPUs":[{"name":"GPU","utilization_percent":61}],"telemetryValid":true,"msSince":137}}`)) + b.nodeInfoNotificationHandler(0)(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","GPUs":[{"name":"GPU","utilization_percent":61}],"telemetryValid":true,"msSince":137}}`)) info := readLocalInfo(t, messages) assert.Equal(t, "owned-host", info.HostUUID) assert.True(t, info.TelemetryValid) @@ -38,7 +38,7 @@ func TestLocalNodeInfo_ReplayPreservesSampleAge(t *testing.T) { // Replaying it must include that wait in its age so an old measurement // cannot appear freshly sampled after scanner restarts. b := &Broker{nodeID: "owned-host"} - b.forwardNodeInfoNotification(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","telemetryValid":true,"msSince":137}}`)) + b.nodeInfoNotificationHandler(0)(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","telemetryValid":true,"msSince":137}}`)) require.NotNil(t, b.localInfo) b.localInfo.receivedAt = time.Now().Add(-3 * time.Second) sp, messages := localInfoScanner(t) @@ -52,10 +52,25 @@ func TestLocalNodeInfo_RejectsForeignWorkerIdentity(t *testing.T) { // A snapshot naming another host must not enter the locally owned sample // cache, even when it reports valid telemetry. b := &Broker{nodeID: "owned-host"} - b.forwardNodeInfoNotification(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"someone-else","telemetryValid":true}}`)) + b.nodeInfoNotificationHandler(0)(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"someone-else","telemetryValid":true}}`)) assert.Nil(t, b.localInfo) } +func TestNodeInfoRestart_RejectsSnapshotsFromPreviousWorker(t *testing.T) { + b := &Broker{nodeID: "owned-host"} + previous := b.nodeInfoNotificationHandler(b.nodeInfoGeneration) + previous(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","telemetryValid":true,"msSince":137}}`)) + require.NotNil(t, b.localInfo) + retained := *b.localInfo + + b.clearNodeInfoState() + previous(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","telemetryValid":true,"msSince":0}}`)) + + assert.Equal(t, retained, *b.localInfo) + b.nodeInfoNotificationHandler(b.nodeInfoGeneration)(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"owned-host","telemetryValid":true,"msSince":42}}`)) + assert.Equal(t, int64(42), b.localInfo.info.MSSince) +} + // localInfoScanner connects the broker's production relay to an in-memory // scanner stand-in. It returns the relay handle and messages received at the // scanner end, and registers cleanup to stop the relay and close both ends. @@ -304,6 +319,42 @@ func TestPeerMetadata_CoalescesUpdatesWhileWorkerIsBlocked(t *testing.T) { }, got) } +func TestPeerMetadata_ReplaysLatestSnapshotAfterBlockedWorkerIsReplaced(t *testing.T) { + // Closing the old stdin must release the sole sender so restart replay can + // deliver updates that arrived while the retired worker was blocked. + b := &Broker{nodeID: "owned-host"} + client, server := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, client.Close()) + assert.NoError(t, server.Close()) + }) + writer := &blockedMetadataWriter{Conn: client, started: make(chan struct{})} + b.setNodeInfo(&nodeInfoProcess{stdin: writer}) + node := noderec.DirectoryNode{HostUUID: "owned-host", IdentityVerification: noderec.IdentityLocal, Name: "initial"} + b.updateLocalPeerMetadata(node) + select { + case <-writer.started: + case <-time.After(2 * time.Second): + require.FailNow(t, "metadata write did not start") + } + node.Name = "latest" + b.updateLocalPeerMetadata(node) + b.clearNodeInfoState() + replacement := &nodeInfoFrameWriter{frames: make(chan []byte, 4)} + b.setNodeInfo(&nodeInfoProcess{stdin: replacement}) + b.pushPeerMetadata() + + require.NoError(t, client.Close()) + waitForMetadataSender(t, b) + + require.Len(t, replacement.frames, 1) + var frame Message + require.NoError(t, json.Unmarshal(<-replacement.frames, &frame)) + var got noderec.PeerMetadataParams + require.NoError(t, json.Unmarshal(frame.Params, &got)) + assert.Equal(t, "latest", got.Peer.Name) +} + func TestPeerMetadata_ClosingWorkerPipeReleasesSender(t *testing.T) { // Shutdown closes stdin without taking its write lock, so a blocked metadata // write and its pending update must both finish rather than leave a sender. diff --git a/services/nvpair-ui-broker/proxy.go b/services/nvpair-ui-broker/proxy.go index 1d9a27de3..a2d5c71c8 100644 --- a/services/nvpair-ui-broker/proxy.go +++ b/services/nvpair-ui-broker/proxy.go @@ -64,6 +64,10 @@ type proxyProcess struct { relayDir *relay.Directory subMu sync.Mutex subIDs map[string]int + // pendingNodes retains only the latest complete snapshot per engine while + // one sender writes to stdin. Both fields are protected by subMu. + pendingNodes map[string][]noderec.DirectoryNode + sendingNodes bool } // proxyReadyParams mirrors the proxy's "ready" notification payload: a facade @@ -161,6 +165,7 @@ func startProxy(name, binaryPath, logLevel string, relayDir *relay.Directory, on } delete(pp.subIDs, engine) } + pp.pendingNodes = nil pp.subMu.Unlock() close(pp.done) }() @@ -226,10 +231,20 @@ func (p *proxyProcess) handleSubscribe(engine string, params json.RawMessage) { if p.relayDir == nil { return } + var id int // read and assigned under subMu, including in the callback send := func(nodes []noderec.DirectoryNode) { - method := engines.AddressMethod(engine, noderec.NotifyNodes) - if err := p.peer.Notify(method, noderec.GetNodesResult{Nodes: nodes}); err != nil { - slog.Debug("failed to push node snapshot to proxy", "engine", engine, "err", err) + p.subMu.Lock() + defer p.subMu.Unlock() + if p.subIDs[engine] != id { + return // a delivery from a replaced subscription must not overwrite it + } + if p.pendingNodes == nil { + p.pendingNodes = make(map[string][]noderec.DirectoryNode) + } + p.pendingNodes[engine] = nodes + if !p.sendingNodes { + p.sendingNodes = true + go p.sendNodeSnapshots() } } p.subMu.Lock() @@ -239,7 +254,10 @@ func (p *proxyProcess) handleSubscribe(engine string, params json.RawMessage) { if prior := p.subIDs[engine]; prior != 0 { p.relayDir.Unsubscribe(prior) } - id, sub, err := subscribeRelay(p.relayDir, params, send) + delete(p.pendingNodes, engine) + var sub *relay.Subscriber + var err error + id, sub, err = subscribeRelay(p.relayDir, params, send) p.subIDs[engine] = id p.subMu.Unlock() if err != nil { @@ -249,6 +267,32 @@ func (p *proxyProcess) handleSubscribe(engine string, params json.RawMessage) { p.relayDir.Deliver(sub) } +// The relay serializes initial and live deliveries before they are queued. A +// blocked pipe retains one pending snapshot per engine, including empty sets, +// while the writer runs outside discovery, relay, and subscription locks. +func (p *proxyProcess) sendNodeSnapshots() { + for { + p.subMu.Lock() + engine := "" + var nodes []noderec.DirectoryNode + for candidate, snapshot := range p.pendingNodes { + engine, nodes = candidate, snapshot + delete(p.pendingNodes, candidate) + break + } + if engine == "" { + p.sendingNodes = false + p.subMu.Unlock() + return + } + p.subMu.Unlock() + method := engines.AddressMethod(engine, noderec.NotifyNodes) + if err := p.peer.Notify(method, noderec.GetNodesResult{Nodes: nodes}); err != nil { + slog.Debug("failed to push node snapshot to proxy", "engine", engine, "err", err) + } + } +} + // Call issues an id-bearing control-plane request to the proxy and blocks until // the response arrives, ctx is cancelled, the proxy exits, or proxyCallTimeout // elapses. diff --git a/services/nvpair-ui-broker/proxyaddressing_test.go b/services/nvpair-ui-broker/proxyaddressing_test.go index 436b40b79..13eed6a9b 100644 --- a/services/nvpair-ui-broker/proxyaddressing_test.go +++ b/services/nvpair-ui-broker/proxyaddressing_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/engines" "nvpair-shared/noderec" "nvpair-ui-broker/relay" @@ -388,3 +390,186 @@ func TestFacadeMethodFor_StripsOnlyItsOwnEngine(t *testing.T) { test("addressed error becomes a bare errors method", ollamaProxyProfile, ollamaProxyProfile.addressed(methodErrorsReport), methodErrorsReport, true) test("unaddressed error is untouched", lmstudioProxyProfile, methodErrorsReport, methodErrorsReport, true) } + +// A proxy with a full stdin pipe must not hold discovery processing or prevent +// pin withdrawal. After recovery it needs the latest empty routing snapshot, +// rather than the intermediate records accumulated while it was not reading. +func TestProxyDiscovery_CoalescesWhilePipeIsBlocked(t *testing.T) { + brokerSide, proxySide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, brokerSide.Close()) + assert.NoError(t, proxySide.Close()) + }) + require.NoError(t, proxySide.SetDeadline(time.Now().Add(6*time.Second))) + writer := &observedProxyWriter{Conn: brokerSide, writing: make(chan struct{})} + b := newManualTestBroker() + b.nodeID, b.clusterDir = "self", t.TempDir() + clustertrusttest.Join(t, b.clusterDir, "cluster", "self", "peer") + mesh := clustertrust.Open(b.clusterDir) + b.relayDir = relay.NewDirectory() + scanner := &scannerProcess{store: b.store, relayDir: b.relayDir, stateMu: &b.discoveryMu, mesh: mesh} + node := noderec.DirectoryNode{ + HostUUID: "peer", Name: "initial", DiscoveryRecordID: mesh.PeerRecordID("peer"), + IdentityVerification: noderec.IdentityPinned, Trusted: true, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + } + initialEvent, err := json.Marshal(noderec.NodeEvent{Node: node}) + require.NoError(t, err) + scanner.handleNotify(noderec.NotifyNodeDiscovered, initialEvent) + proxy := &proxyProcess{peer: NewPeer(NewCodec(writer)), relayDir: b.relayDir} + params, err := json.Marshal(noderec.SubscribeParams{Services: []noderec.ServiceKey{noderec.ServiceOllama}}) + require.NoError(t, err) + subscribed := make(chan struct{}) + go func() { + proxy.handleSubscribe("ollama", params) + close(subscribed) + }() + select { + case <-writer.writing: + case <-time.After(time.Second): + require.FailNow(t, "the initial snapshot was not sent") + } + finished := func(done <-chan struct{}, operation string) { + t.Helper() + select { + case <-done: + case <-time.After(time.Second): + assert.Fail(t, operation+" waited for the blocked proxy") + } + } + node.Name = "intermediate" + updatedEvent, err := json.Marshal(noderec.NodeEvent{Node: node}) + require.NoError(t, err) + scanned := make(chan struct{}) + go func() { + for range 20 { + scanner.handleNotify(noderec.NotifyNodeUpdated, updatedEvent) + } + close(scanned) + }() + finished(scanned, "scanner update") + owned := make(chan struct{}) + go func() { + b.nodeInfoNotificationHandler(0)(noderec.NotifyInfoUpdated, json.RawMessage(`{"info":{"hostUuid":"self"}}`)) + close(owned) + }() + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + withdrawn := make(chan struct{}) + go func() { + b.reconcileDiscoveryTrust() + close(withdrawn) + }() + finished(owned, "local hardware update") + finished(withdrawn, "trust withdrawal") + finished(subscribed, "subscription setup") + + codec := NewCodec(proxySide) + initial, err := codec.Read() + require.NoError(t, err) + require.Equal(t, "ollama:discovery:nodes", initial.Method) + var first noderec.GetNodesResult + require.NoError(t, json.Unmarshal(initial.Params, &first)) + require.Len(t, first.Nodes, 1) + assert.Equal(t, "initial", first.Nodes[0].Name) + last, err := codec.Read() + require.NoError(t, err) + assert.Equal(t, "ollama:discovery:nodes", last.Method) + var final noderec.GetNodesResult + require.NoError(t, json.Unmarshal(last.Params, &final)) + assert.NotNil(t, final.Nodes, "an empty authoritative snapshot is encoded as []") + assert.Empty(t, final.Nodes) + finished(scanned, "scanner after pipe recovery") + finished(owned, "local hardware after pipe recovery") + finished(withdrawn, "trust withdrawal after pipe recovery") + waitForProxySnapshotSender(t, proxy) +} + +func waitForProxySnapshotSender(t *testing.T, proxy *proxyProcess) { + t.Helper() + // An idle sender also proves that no extra old snapshot remains blocked on + // the pipe after the expected complete snapshots have been consumed. + require.Eventually(t, func() bool { + proxy.subMu.Lock() + defer proxy.subMu.Unlock() + return !proxy.sendingNodes + }, time.Second, time.Millisecond) +} + +// One facade changing its filter must not discard another facade's pending +// update, or let its previous subscription overwrite the replacement's set. +func TestProxyDiscovery_KeepsEachEnginesLatestSubscriptionSnapshot(t *testing.T) { + brokerSide, proxySide := net.Pipe() + t.Cleanup(func() { + assert.NoError(t, brokerSide.Close()) + assert.NoError(t, proxySide.Close()) + }) + require.NoError(t, proxySide.SetDeadline(time.Now().Add(3*time.Second))) + writer := &observedProxyWriter{Conn: brokerSide, writing: make(chan struct{})} + directory := relay.NewDirectory() + node := noderec.DirectoryNode{ + HostUUID: "peer", DiscoveryRecordID: "peer:peer", Name: "initial", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceOllama: {Port: 11434}, noderec.ServiceLMStudio: {Port: 1234}, + }, + } + directory.Apply(noderec.NotifyNodeDiscovered, node) + proxy := &proxyProcess{peer: NewPeer(NewCodec(writer)), relayDir: directory} + params := func(service noderec.ServiceKey) json.RawMessage { + t.Helper() + raw, err := json.Marshal(noderec.SubscribeParams{Services: []noderec.ServiceKey{service}}) + require.NoError(t, err) + return raw + } + ollamaParams, lmstudioParams, nodeInfoParams := params(noderec.ServiceOllama), params(noderec.ServiceLMStudio), params(noderec.ServiceNodeInfo) + subscribed := make(chan struct{}) + go func() { + proxy.handleSubscribe("ollama", ollamaParams) + close(subscribed) + }() + select { + case <-writer.writing: + case <-time.After(time.Second): + require.FailNow(t, "the initial snapshot was not sent") + } + select { + case <-subscribed: + case <-time.After(time.Second): + require.FailNow(t, "subscription setup waited for the blocked writer") + } + node.Name = "latest" + updated := make(chan struct{}) + go func() { + proxy.handleSubscribe("lmstudio", lmstudioParams) + proxy.handleSubscribe("ollama", nodeInfoParams) + directory.Apply(noderec.NotifyNodeUpdated, node) + close(updated) + }() + select { + case <-updated: + case <-time.After(time.Second): + require.FailNow(t, "updating subscriptions waited for the blocked writer") + } + + codec := NewCodec(proxySide) + initial, err := codec.Read() + require.NoError(t, err) + assert.Equal(t, "ollama:discovery:nodes", initial.Method) + var first noderec.GetNodesResult + require.NoError(t, json.Unmarshal(initial.Params, &first)) + require.Len(t, first.Nodes, 1) + assert.Equal(t, "initial", first.Nodes[0].Name) + snapshots := make(map[string][]noderec.DirectoryNode) + for range 2 { + message, err := codec.Read() + require.NoError(t, err) + require.NotContains(t, snapshots, message.Method) + var result noderec.GetNodesResult + require.NoError(t, json.Unmarshal(message.Params, &result)) + snapshots[message.Method] = result.Nodes + } + assert.Equal(t, map[string][]noderec.DirectoryNode{ + "ollama:discovery:nodes": {}, + "lmstudio:discovery:nodes": {node}, + }, snapshots) + waitForProxySnapshotSender(t, proxy) +} diff --git a/services/nvpair-ui-broker/telemetry.go b/services/nvpair-ui-broker/telemetry.go index a221daed6..260f67e35 100644 --- a/services/nvpair-ui-broker/telemetry.go +++ b/services/nvpair-ui-broker/telemetry.go @@ -162,6 +162,9 @@ func (b *Broker) removeTelemetry(source nodeSource, hostUUID string) { func (b *Broker) fanTelemetryToScheduler(telemetry noderec.NodeTelemetry, projectedAt time.Time) { b.schedulerFeedMu.Lock() defer b.schedulerFeedMu.Unlock() + if telemetry.TelemetryValid && b.clusterDir != "" && !b.currentTelemetrySource(telemetry.HostUUID) { + telemetry = noderec.NodeTelemetry{HostUUID: telemetry.HostUUID} + } scheduler := b.getScheduler() if scheduler == nil { @@ -181,7 +184,12 @@ func (b *Broker) replayTelemetryToScheduler(scheduler *rpcWorker) int { return 0 } replayed := 0 - for _, telemetry := range b.telemetry.Snapshot(time.Now()) { + snapshotAt := time.Now() + for _, telemetry := range b.telemetry.Snapshot(snapshotAt) { + if b.clusterDir != "" && !b.currentTelemetrySource(telemetry.HostUUID) { + continue + } + telemetry = telemetry.Aged(time.Since(snapshotAt)) if err := scheduler.Notify(schedulerwire.MethodTelemetry, telemetry); err != nil { slog.Warn("replay telemetry to scheduler failed", "host_uuid", telemetry.HostUUID, "err", err) break @@ -190,3 +198,19 @@ func (b *Broker) replayTelemetryToScheduler(scheduler *rpcWorker) int { } return replayed } + +// currentTelemetrySource checks the source the cache would project, rather +// than allowing a different, surviving source to authorize its old sample. +func (b *Broker) currentTelemetrySource(uuid string) bool { + b.telemetry.mu.RLock() + sources := b.telemetry.nodes[uuid] + source := sourceManual + if sources.scanner != nil { + source = sourceScanner + } + b.telemetry.mu.RUnlock() + mesh := b.currentDiscoveryMesh() + mesh.Refresh() + node, present := b.store.sourceNode(uuid, source) + return present && (node.IdentityVerification == noderec.IdentityLocal || (node.IdentityVerification == noderec.IdentityPinned && node.Trusted && node.DiscoveryRecordID != "" && node.DiscoveryRecordID == mesh.PeerRecordID(uuid))) +} diff --git a/services/nvpair-ui-broker/telemetry_test.go b/services/nvpair-ui-broker/telemetry_test.go index cf19adeae..0d76b46a7 100644 --- a/services/nvpair-ui-broker/telemetry_test.go +++ b/services/nvpair-ui-broker/telemetry_test.go @@ -13,10 +13,70 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" "nvpair-shared/schedulerwire" ) +func TestTelemetryReplay_DiscardsSamplesWhosePinWasRemoved(t *testing.T) { + b := newManualTestBroker() + b.clusterDir = t.TempDir() + clustertrusttest.Join(t, b.clusterDir, "cluster", "self", "peer") + mesh := clustertrust.Open(b.clusterDir) + b.store.Upsert(EnrichedNode{HostUUID: "peer", DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, Trusted: true}, sourceScanner) + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", TelemetryValid: true}) + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + var output bytes.Buffer + scheduler := &rpcWorker{peer: NewPeer(NewCodec(&output))} + + b.schedulerFeedMu.Lock() + replayed := b.replayTelemetryToScheduler(scheduler) + b.schedulerFeedMu.Unlock() + + assert.Zero(t, replayed) + assert.Empty(t, output.Bytes()) +} + +type delayedTelemetryWriter struct { + bytes.Buffer + delay time.Duration + writes int +} + +func (w *delayedTelemetryWriter) Write(data []byte) (int, error) { + w.writes++ + if w.writes == 1 { + time.Sleep(w.delay) + } + return w.Buffer.Write(data) +} + +func TestTelemetryReplay_IncludesTimeWaitingBehindEarlierSamples(t *testing.T) { + const delay = 100 * time.Millisecond + b := newManualTestBroker() + now := time.Now() + b.telemetry.Upsert(sourceScanner, noderec.NodeTelemetry{HostUUID: "first", TelemetryValid: true}, now, now) + b.telemetry.Upsert(sourceScanner, noderec.NodeTelemetry{HostUUID: "second", TelemetryValid: true}, now, now) + output := &delayedTelemetryWriter{delay: delay} + scheduler := &rpcWorker{peer: NewPeer(NewCodec(output))} + + b.schedulerFeedMu.Lock() + replayed := b.replayTelemetryToScheduler(scheduler) + b.schedulerFeedMu.Unlock() + + require.Equal(t, 2, replayed) + codec := NewCodec(output) + _, err := codec.Read() + require.NoError(t, err) + message, err := codec.Read() + require.NoError(t, err) + var sample noderec.NodeTelemetry + require.NoError(t, json.Unmarshal(message.Params, &sample)) + assert.Equal(t, "second", sample.HostUUID) + assert.GreaterOrEqual(t, sample.MSSince, delay.Milliseconds()) +} + func TestTelemetryCache_IncludesElapsedTimeInObservationAge(t *testing.T) { cache := newTelemetryCache() receivedAt := time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant diff --git a/services/shared/clustertrust/clustertrust.go b/services/shared/clustertrust/clustertrust.go index c924a16f9..01748c77d 100644 --- a/services/shared/clustertrust/clustertrust.go +++ b/services/shared/clustertrust/clustertrust.go @@ -75,14 +75,27 @@ func LoadIdentity(clusterDir string) (*Identity, error) { type Trust struct { dir string mu sync.RWMutex - ders map[string][]byte + pins map[string]trustedPin +} + +type trustedPin struct { + der []byte + recordKey string +} + +type peerPinRecord struct { + NodeUUID string `json:"nodeUuid"` + CertPem string `json:"certPem"` + ClusterID string `json:"clusterId"` + AdmissionEpoch uint64 `json:"admissionEpoch"` + PinnedAt int64 `json:"pinnedAt"` } // newTrust opens clusterDir/trusted. The pins are read by the owning Mesh's // first Refresh, not here, so there is one place that decides when the cluster // dir is read. func newTrust(clusterDir string) *Trust { - return &Trust{dir: filepath.Join(clusterDir, "trusted"), ders: map[string][]byte{}} + return &Trust{dir: filepath.Join(clusterDir, "trusted"), pins: map[string]trustedPin{}} } // Reload re-reads the trusted/ directory, replacing the in-memory pin set. @@ -107,7 +120,7 @@ func (t *Trust) Reload() { } // A missing dir leaves entries nil, so the loop below is a no-op and the // empty set is published — the authoritative "no peers trusted". - ders := make(map[string][]byte, len(entries)) + pins := make(map[string]trustedPin, len(entries)) for _, e := range entries { name := e.Name() if e.IsDir() || !strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".tmp") { @@ -121,15 +134,12 @@ func (t *Trust) Reload() { // dir won't read: the file's presence says the peer is still // trusted, so failing to open it is missing information rather than // a revocation. A pin we have never read stays absent. - if previous, ok := t.DER(fileUUID); ok { - ders[fileUUID] = previous + if previous, ok := t.pin(fileUUID); ok { + pins[fileUUID] = previous } continue } - var pin struct { - NodeUUID string `json:"nodeUuid"` - CertPem string `json:"certPem"` - } + var pin peerPinRecord // A pin that reads but does not parse or does not validate is dropped, // not carried: that is a content problem, and honoring a certificate we // can no longer verify the provenance of is the one failure this store @@ -141,13 +151,21 @@ func (t *Trust) Reload() { if err != nil { continue } - ders[fileUUID] = der + pins[fileUUID] = trustedPin{der: der, recordKey: fmt.Sprintf("%s:%d:%d", pin.ClusterID, pin.AdmissionEpoch, pin.PinnedAt)} } t.mu.Lock() - t.ders = ders + t.pins = pins t.mu.Unlock() } +// pin returns one immutable certificate-and-generation snapshot. +func (t *Trust) pin(uuid string) (trustedPin, bool) { + t.mu.RLock() + defer t.mu.RUnlock() + pin, ok := t.pins[uuid] + return pin, ok +} + // pinDER validates that a pin file's inner nodeUuid, filename, and embedded // certificate principal all agree (so a renamed/tampered file can't masquerade // as another UUID) and returns the certificate DER. Mirrors @@ -172,31 +190,25 @@ func pinDER(innerUUID, certPem, fileUUID string) ([]byte, error) { // MatchDER reports whether der byte-for-byte matches the pinned cert for uuid. func (t *Trust) MatchDER(uuid string, der []byte) bool { - t.mu.RLock() - defer t.mu.RUnlock() - pinned, ok := t.ders[uuid] - return ok && bytes.Equal(pinned, der) + pin, ok := t.pin(uuid) + return ok && bytes.Equal(pin.der, der) } // DER returns a copy of the pinned cert DER for uuid, used to pin a peer's // server cert on an outbound call. func (t *Trust) DER(uuid string) ([]byte, bool) { - t.mu.RLock() - defer t.mu.RUnlock() - der, ok := t.ders[uuid] + pin, ok := t.pin(uuid) if !ok { return nil, false } - cp := make([]byte, len(der)) - copy(cp, der) - return cp, true + return bytes.Clone(pin.der), true } // Count returns the number of currently-pinned peers. func (t *Trust) Count() int { t.mu.RLock() defer t.mu.RUnlock() - return len(t.ders) + return len(t.pins) } // ServerTLSConfig builds the inbound mTLS config from a node's leaf: present it diff --git a/services/shared/clustertrust/clustertrust_test.go b/services/shared/clustertrust/clustertrust_test.go index d64fd650c..8e69cfb0f 100644 --- a/services/shared/clustertrust/clustertrust_test.go +++ b/services/shared/clustertrust/clustertrust_test.go @@ -112,6 +112,25 @@ func TestTrustPin_AuthorizesOnlyMatchingCertificate(t *testing.T) { require.Len(t, sc.Certificates, 1, "server config: ClientAuth") } +func TestTrustDER_ReturnsIndependentCertificateBytes(t *testing.T) { + clusterDir := t.TempDir() + certPEM, _, expected := genLeaf(t, "uuid-peer") + writePin(t, clusterDir, "uuid-peer", string(certPEM)) + trust := newTrust(clusterDir) + trust.Reload() + der, ok := trust.DER("uuid-peer") + require.True(t, ok) + require.Equal(t, expected, der) + + // A caller changing its returned bytes must not change the saved pin or + // the certificate another caller receives. + der[0] ^= 0xff + assert.True(t, trust.MatchDER("uuid-peer", expected)) + next, ok := trust.DER("uuid-peer") + require.True(t, ok) + assert.Equal(t, expected, next) +} + func TestVerifyClientPin_AcceptsPinnedCertificateAndRejectsUnpinnedClient(t *testing.T) { dir := t.TempDir() peerCertPEM, _, _ := genLeaf(t, "uuid-peer") diff --git a/services/shared/clustertrust/membership.go b/services/shared/clustertrust/membership.go index 74c3edad6..678cc6034 100644 --- a/services/shared/clustertrust/membership.go +++ b/services/shared/clustertrust/membership.go @@ -25,25 +25,24 @@ type admissionRecord struct { Epoch uint64 `json:"epoch"` } -// hasActiveAdmission reports whether clusterDir/admission.json records a live -// admission. A missing, unreadable, malformed, or cleared file reads as false -// (not a member). This is the durable, restart-surviving membership fact, in -// contrast to keypair presence which outlives a membership by design. -// -// It is re-read on every Mesh.Refresh rather than cached: membership is exactly -// the fact that changes underneath a running service when the user creates, -// joins, or leaves a cluster. -func hasActiveAdmission(clusterDir string) bool { +// readAdmission returns the durable membership and epoch, or an empty record +// without an error when the file is absent or malformed. Other filesystem errors +// are returned so callers can retain the last known admission during read +// failures. The epoch distinguishes a rejoin using the previous certificate. +func readAdmission(clusterDir string) (admissionRecord, error) { if clusterDir == "" { - return false + return admissionRecord{}, nil } data, err := os.ReadFile(filepath.Join(clusterDir, admissionFileName)) if err != nil { - return false + if os.IsNotExist(err) { + return admissionRecord{}, nil + } + return admissionRecord{}, err } var rec admissionRecord if err := json.Unmarshal(data, &rec); err != nil { - return false + return admissionRecord{}, nil } - return rec.ClusterID != "" && rec.Epoch != 0 + return rec, nil } diff --git a/services/shared/clustertrust/mesh.go b/services/shared/clustertrust/mesh.go index 167fc0f42..287b681fe 100644 --- a/services/shared/clustertrust/mesh.go +++ b/services/shared/clustertrust/mesh.go @@ -5,6 +5,7 @@ package clustertrust import ( "bytes" + "crypto/sha256" "crypto/tls" "crypto/x509" "errors" @@ -71,6 +72,9 @@ type Mesh struct { // admission.json, trusted/). Empty means this service was given no cluster // dir and is unclustered for its whole life. dir string + // Serialize disk reloads so a read started before revocation cannot finish + // after a newer refresh and overwrite its view with older pins. + refreshMu sync.Mutex // mu guards the identity and the derived membership answer. trust carries // its own lock and is replaced only in Open. @@ -78,6 +82,7 @@ type Mesh struct { id *Identity idCertPEM []byte clustered bool + admission admissionRecord trust *Trust } @@ -120,6 +125,8 @@ func (m *Mesh) Refresh() { if m == nil || m.dir == "" { return } + m.refreshMu.Lock() + defer m.refreshMu.Unlock() m.trust.Reload() certPEM, certErr := os.ReadFile(filepath.Join(m.dir, "node.crt")) @@ -134,11 +141,42 @@ func (m *Mesh) Refresh() { m.id, m.idCertPEM = id, certPEM } } + // Keep the previous admission on a read failure. An absent, cleared, or + // malformed record still replaces it, so departures remain authoritative. + if admission, err := readAdmission(m.dir); err == nil { + m.admission = admission + } // Membership is a usable identity AND either a live admission (authoritative // and durable across restarts, including a cluster of one with no peers) or // at least one pinned peer (a confirmed peer means clustered even on a // cluster dir written before admission.json existed). - m.clustered = m.id != nil && (hasActiveAdmission(m.dir) || m.trust.Count() > 0) + m.clustered = m.id != nil && ((m.admission.ClusterID != "" && m.admission.Epoch != 0) || m.trust.Count() > 0) +} + +// PeerRecordID identifies a proof made with the current local identity, +// admission, and exact peer pin. It is an opaque discovery record key, not an +// authenticator: only an owned worker's successful mTLS read can publish it. +// Call Refresh first. Empty means the principal is currently unauthorized. +func (m *Mesh) PeerRecordID(uuid string) string { + if m == nil || uuid == "" { + return "" + } + m.mu.RLock() + defer m.mu.RUnlock() + if !m.clustered || m.id == nil { + return "" + } + der := selfLeafDER(m.id) + peerKey := "" + if uuid != m.id.NodeUUID { + pin, ok := m.trust.pin(uuid) + if !ok { + return "" + } + der, peerKey = pin.der, ":"+pin.recordKey + } + key := fmt.Sprintf("%x:%x:%s:%d%s", sha256.Sum256(selfLeafDER(m.id)), sha256.Sum256(der), m.admission.ClusterID, m.admission.Epoch, peerKey) + return fmt.Sprintf("peer:%s:%x", uuid, sha256.Sum256([]byte(key))) } // Clustered reports whether this node is CURRENTLY a cluster member, as of the diff --git a/services/shared/clustertrust/mesh_test.go b/services/shared/clustertrust/mesh_test.go index 74cab6336..ff53973a9 100644 --- a/services/shared/clustertrust/mesh_test.go +++ b/services/shared/clustertrust/mesh_test.go @@ -6,6 +6,7 @@ package clustertrust import ( "crypto/tls" "crypto/x509" + "encoding/json" "net/http" "net/http/httptest" "os" @@ -25,25 +26,178 @@ func writeIdentity(t *testing.T, dir string, certPEM, keyPEM []byte) { } func TestMesh_VerifyPeerCertificateUsesCurrentExactPin(t *testing.T) { - selfPEM, selfKey, _ := genLeaf(t, "self") - peerPEM, _, peerDER := genLeaf(t, "peer") - replacementPEM, _, replacementDER := genLeaf(t, "peer") - mesh := loadMeshDir(t, selfPEM, selfKey, map[string]string{"peer": string(peerPEM)}) + const selfUUID = "self" + const peerUUID = "peer" + + selfPEM, selfKey, _ := genLeaf(t, selfUUID) + peerPEM, _, peerDER := genLeaf(t, peerUUID) + replacementPEM, _, replacementDER := genLeaf(t, peerUUID) + mesh := loadMeshDir(t, selfPEM, selfKey, map[string]string{peerUUID: string(peerPEM)}) peer, err := x509.ParseCertificate(peerDER) require.NoError(t, err) replacement, err := x509.ParseCertificate(replacementDER) require.NoError(t, err) - require.True(t, mesh.VerifyPeerCertificate("peer", peer)) + require.True(t, mesh.VerifyPeerCertificate(peerUUID, peer)) assert.False(t, mesh.VerifyPeerCertificate("another-peer", peer)) - assert.False(t, mesh.VerifyPeerCertificate("peer", replacement)) - assert.False(t, mesh.VerifyPeerCertificate("peer", nil)) - writePin(t, mesh.dir, "peer", string(replacementPEM)) + assert.False(t, mesh.VerifyPeerCertificate(peerUUID, replacement)) + assert.False(t, mesh.VerifyPeerCertificate(peerUUID, nil)) + writePin(t, mesh.dir, peerUUID, string(replacementPEM)) + mesh.Refresh() + assert.False(t, mesh.VerifyPeerCertificate(peerUUID, peer)) + require.True(t, mesh.VerifyPeerCertificate(peerUUID, replacement)) + require.NoError(t, os.Remove(filepath.Join(mesh.dir, "trusted", peerUUID+".json"))) + mesh.Refresh() + assert.False(t, mesh.VerifyPeerCertificate(peerUUID, replacement)) +} + +// A record ID binds an authenticated result to the current authorization, so +// cached results cannot survive membership or certificate changes. Unchanged +// authorization keeps the ID stable, and removing authorization leaves no ID. +func TestMesh_PeerRecordIDChangesWithAuthorization(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const clusterID = "cluster" + + selfPEM, selfKey, _ := genLeaf(t, selfUUID) + peerPEM, _, _ := genLeaf(t, peerUUID) + mesh := loadMeshDir(t, selfPEM, selfKey, map[string]string{peerUUID: string(peerPEM)}) + previous := mesh.PeerRecordID(peerUUID) + require.NotEmpty(t, previous) + assert.Empty(t, mesh.PeerRecordID("unapproved")) + // Reloading unchanged trust must not invalidate an authenticated result. + mesh.Refresh() + assert.Equal(t, previous, mesh.PeerRecordID(peerUUID)) + + // A new local admission invalidates earlier results, even when both nodes + // keep their certificates. Advancing the epoch models leaving and rejoining. + writeAdmission(t, mesh.dir, clusterID, 1) + mesh.Refresh() + admitted := mesh.PeerRecordID(peerUUID) + assert.NotEqual(t, previous, admitted) + writeAdmission(t, mesh.dir, clusterID, 2) + mesh.Refresh() + assert.NotEqual(t, admitted, mesh.PeerRecordID(peerUUID)) + + // Replacing the local certificate also invalidates earlier peer results. + previous = mesh.PeerRecordID(peerUUID) + replacementPEM, replacementKey, _ := genLeaf(t, selfUUID) + writeIdentity(t, mesh.dir, replacementPEM, replacementKey) + mesh.Refresh() + assert.NotEqual(t, previous, mesh.PeerRecordID(peerUUID)) + + // Re-admitting the peer starts a new authorization even if its UUID and + // certificate are unchanged. The saved admission details must affect the ID. + previous = mesh.PeerRecordID(peerUUID) + type savedPin struct { + NodeUUID string `json:"nodeUuid"` + CertPEM string `json:"certPem"` + AdmissionEpoch uint64 `json:"admissionEpoch"` + PinnedAt int64 `json:"pinnedAt"` + } + pinPath := filepath.Join(mesh.dir, "trusted", peerUUID+".json") + data, err := os.ReadFile(pinPath) + require.NoError(t, err) + var pin savedPin + require.NoError(t, json.Unmarshal(data, &pin)) + pin.AdmissionEpoch = 2 + pin.PinnedAt = 1000 + data, err = json.Marshal(pin) + require.NoError(t, err) + require.NoError(t, os.WriteFile(pinPath, data, 0o600)) + mesh.Refresh() + assert.NotEqual(t, previous, mesh.PeerRecordID(peerUUID)) + + // Replacing the peer's certificate invalidates its earlier results. Removing + // its pin then leaves the peer unauthorized, so it must have no record ID. + previous = mesh.PeerRecordID(peerUUID) + replacementPeerPEM, _, _ := genLeaf(t, peerUUID) + writePin(t, mesh.dir, peerUUID, string(replacementPeerPEM)) mesh.Refresh() - assert.False(t, mesh.VerifyPeerCertificate("peer", peer)) - require.True(t, mesh.VerifyPeerCertificate("peer", replacement)) - require.NoError(t, os.Remove(filepath.Join(mesh.dir, "trusted", "peer.json"))) + assert.NotEqual(t, previous, mesh.PeerRecordID(peerUUID)) + require.NoError(t, os.Remove(pinPath)) mesh.Refresh() - assert.False(t, mesh.VerifyPeerCertificate("peer", replacement)) + assert.Empty(t, mesh.PeerRecordID(peerUUID)) +} + +// A failed admission read is missing information, not a membership change. +// Existing local membership and peer record IDs must survive until a later +// successful read can establish whether authorization changed. +func TestMesh_PreservesAuthorizationWhenAdmissionReadFails(t *testing.T) { + const selfUUID = "self" + const peerUUID = "peer" + const clusterID = "cluster" + + test := func(name, principal string, pins map[string]string) { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + selfPEM, selfKey, _ := genLeaf(t, selfUUID) + writeIdentity(t, dir, selfPEM, selfKey) + writeAdmission(t, dir, clusterID, 1) + for uuid, certPEM := range pins { + writePin(t, dir, uuid, certPEM) + } + mesh := Open(dir) + require.True(t, mesh.Clustered()) + previous := mesh.PeerRecordID(principal) + require.NotEmpty(t, previous) + + // A directory at the file path makes ReadFile fail on every platform, + // without relying on permission changes that Windows may ignore. + path := filepath.Join(mesh.dir, admissionFileName) + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0o700)) + mesh.Refresh() + + assert.True(t, mesh.Clustered()) + assert.Equal(t, previous, mesh.PeerRecordID(principal)) + + // Once reads recover, a new admission must invalidate the old ID. + require.NoError(t, os.Remove(path)) + writeAdmission(t, dir, clusterID, 2) + mesh.Refresh() + assert.True(t, mesh.Clustered()) + assert.NotEqual(t, previous, mesh.PeerRecordID(principal)) + }) + } + + peerPEM, _, _ := genLeaf(t, peerUUID) + test("local membership without peers", selfUUID, nil) + test("verified peer record", peerUUID, map[string]string{peerUUID: string(peerPEM)}) +} + +func TestMesh_ClearsAdmissionWhenFileIsAbsentOrInvalid(t *testing.T) { + const selfUUID = "self" + const clusterID = "cluster" + + test := func(name string, change func(*testing.T, string)) { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + selfPEM, selfKey, _ := genLeaf(t, selfUUID) + writeIdentity(t, dir, selfPEM, selfKey) + writeAdmission(t, dir, clusterID, 1) + mesh := Open(dir) + require.True(t, mesh.Clustered()) + + change(t, dir) + mesh.Refresh() + + assert.False(t, mesh.Clustered()) + assert.Empty(t, mesh.PeerRecordID(selfUUID)) + }) + } + + test("admission file removed", func(t *testing.T, dir string) { + require.NoError(t, os.Remove(filepath.Join(dir, admissionFileName))) + }) + test("admission explicitly cleared", func(t *testing.T, dir string) { + writeAdmission(t, dir, "", 0) + }) + test("malformed JSON", func(t *testing.T, dir string) { + require.NoError(t, os.WriteFile(filepath.Join(dir, admissionFileName), []byte(`{"clusterId":`), 0o600)) + }) + test("invalid field type", func(t *testing.T, dir string) { + require.NoError(t, os.WriteFile(filepath.Join(dir, admissionFileName), []byte(`{"clusterId":"cluster","epoch":"invalid"}`), 0o600)) + }) } func loadMeshDir(t *testing.T, certPEM, keyPEM []byte, pins map[string]string) *Mesh { @@ -83,20 +237,24 @@ func TestMeshOpen_LeavesUnclusteredNodesWithoutPeerAuthorization(t *testing.T) { // any-pin client works, and a node that completes the handshake but isn't // pinned by the server is rejected at the gate. func TestMeshGate_RequiresMembershipAndPinnedIdentity(t *testing.T) { - selfPEM, selfKey, _ := genLeaf(t, "uuid-self") - peerPEM, peerKey, _ := genLeaf(t, "uuid-peer") - strangerPEM, strangerKey, _ := genLeaf(t, "uuid-stranger") + const selfUUID = "uuid-self" + const peerUUID = "uuid-peer" + const strangerUUID = "uuid-stranger" + + selfPEM, selfKey, _ := genLeaf(t, selfUUID) + peerPEM, peerKey, _ := genLeaf(t, peerUUID) + strangerPEM, strangerKey, _ := genLeaf(t, strangerUUID) // self pins peer (only). peer + stranger both pin self (so both can complete // a handshake TO self) — but self does not pin stranger. - selfMesh := loadMeshDir(t, selfPEM, selfKey, map[string]string{"uuid-peer": string(peerPEM)}) - peerMesh := loadMeshDir(t, peerPEM, peerKey, map[string]string{"uuid-self": string(selfPEM)}) - strangerMesh := loadMeshDir(t, strangerPEM, strangerKey, map[string]string{"uuid-self": string(selfPEM)}) + selfMesh := loadMeshDir(t, selfPEM, selfKey, map[string]string{peerUUID: string(peerPEM)}) + peerMesh := loadMeshDir(t, peerPEM, peerKey, map[string]string{selfUUID: string(selfPEM)}) + strangerMesh := loadMeshDir(t, strangerPEM, strangerKey, map[string]string{selfUUID: string(selfPEM)}) - assert.Equal(t, "uuid-self", selfMesh.NodeUUID()) - assert.True(t, selfMesh.HasPin("uuid-peer"), "peer must be a pin") - assert.True(t, selfMesh.HasPin("uuid-self"), "self must be trusted (self-trust)") - assert.False(t, selfMesh.HasPin("uuid-stranger"), "stranger must not be a pin") + assert.Equal(t, selfUUID, selfMesh.NodeUUID()) + assert.True(t, selfMesh.HasPin(peerUUID), "peer must be a pin") + assert.True(t, selfMesh.HasPin(selfUUID), "self must be trusted (self-trust)") + assert.False(t, selfMesh.HasPin(strangerUUID), "stranger must not be a pin") // self is the mTLS server, gated by VerifyClientPin. srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -128,23 +286,26 @@ func TestMeshGate_RequiresMembershipAndPinnedIdentity(t *testing.T) { } // Pinned peer -> self, specific pin: accepted. - code, err := do(mustConfig(peerMesh.ClientTLSConfig("uuid-self"))) - require.NoError(t, err, "peer->self (specific pin)") - assert.Equal(t, http.StatusOK, code, "peer->self (specific pin)") + const specificPinContext = "peer->self (specific pin)" + code, err := do(mustConfig(peerMesh.ClientTLSConfig(selfUUID))) + require.NoError(t, err, specificPinContext) + assert.Equal(t, http.StatusOK, code, specificPinContext) // Pinned peer -> self, any-pin (no UUID known up front): accepted. + const anyPinContext = "peer->self (any-pin)" code, err = do(mustConfig(peerMesh.ClientTLSConfigAny())) - require.NoError(t, err, "peer->self (any-pin)") - assert.Equal(t, http.StatusOK, code, "peer->self (any-pin)") + require.NoError(t, err, anyPinContext) + assert.Equal(t, http.StatusOK, code, anyPinContext) // self -> self over mTLS: accepted via self-trust (self isn't in trusted/). - code, err = do(mustConfig(selfMesh.ClientTLSConfig("uuid-self"))) - require.NoError(t, err, "self->self") - assert.Equal(t, http.StatusOK, code, "self->self") + const selfTrustContext = "self->self" + code, err = do(mustConfig(selfMesh.ClientTLSConfig(selfUUID))) + require.NoError(t, err, selfTrustContext) + assert.Equal(t, http.StatusOK, code, selfTrustContext) // Stranger completes the handshake (it pins self) but self doesn't pin it: 403. - code, err = do(mustConfig(strangerMesh.ClientTLSConfig("uuid-self"))) + code, err = do(mustConfig(strangerMesh.ClientTLSConfig(selfUUID))) require.NoError(t, err, "request as unpinned client") assert.Equal(t, http.StatusForbidden, code, "stranger->self") // self cannot even build a client to an unpinned stranger (client-side gate). - _, ok := selfMesh.ClientTLSConfig("uuid-stranger") + _, ok := selfMesh.ClientTLSConfig(strangerUUID) require.False(t, ok, "must not build a client for an unpinned peer") } @@ -155,10 +316,13 @@ func TestMeshGate_RequiresMembershipAndPinnedIdentity(t *testing.T) { // cluster dir is populated a peer handshake is refused; afterwards the very same // listener serves the pinned mTLS surface, with no rebind and no restart. func TestMeshServerTLSConfig_TracksMembershipOnExistingListener(t *testing.T) { + const selfUUID = "uuid-self" + const peerUUID = "uuid-peer" + selfDir := t.TempDir() - selfPEM, selfKey, _ := genLeaf(t, "uuid-self") - peerPEM, peerKey, _ := genLeaf(t, "uuid-peer") - peerMesh := loadMeshDir(t, peerPEM, peerKey, map[string]string{"uuid-self": string(selfPEM)}) + selfPEM, selfKey, _ := genLeaf(t, selfUUID) + peerPEM, peerKey, _ := genLeaf(t, peerUUID) + peerMesh := loadMeshDir(t, peerPEM, peerKey, map[string]string{selfUUID: string(selfPEM)}) selfMesh := Open(selfDir) srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -174,7 +338,7 @@ func TestMeshServerTLSConfig_TracksMembershipOnExistingListener(t *testing.T) { defer srv.Close() get := func() (int, error) { - cfg, ok := peerMesh.ClientTLSConfig("uuid-self") + cfg, ok := peerMesh.ClientTLSConfig(selfUUID) require.True(t, ok, "peer must be able to build a client for self") client := &http.Client{Transport: &http.Transport{TLSClientConfig: cfg}} resp, err := client.Get(srv.URL) @@ -192,13 +356,14 @@ func TestMeshServerTLSConfig_TracksMembershipOnExistingListener(t *testing.T) { // The node joins: identity + admission + the peer's pin land on disk. writeIdentity(t, selfDir, selfPEM, selfKey) writeAdmission(t, selfDir, "cluster-abc", 1) - writePin(t, selfDir, "uuid-peer", string(peerPEM)) + writePin(t, selfDir, peerUUID, string(peerPEM)) selfMesh.Refresh() require.True(t, selfMesh.Clustered(), "the mesh must be clustered once the dir is populated") + const joinedListenerContext = "the same listener must serve the pinned peer after the join" code, err = get() - require.NoError(t, err, "the same listener must serve the pinned peer after the join") - assert.Equal(t, http.StatusOK, code, "the same listener must serve the pinned peer after the join") + require.NoError(t, err, joinedListenerContext) + assert.Equal(t, http.StatusOK, code, joinedListenerContext) } // TestClusterUUIDFromTXT: reads cluster-uuid= and never node-info's host uuid=. From 7830cde4f9ad3365c5da2c749240bba946402be3 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Wed, 21 Oct 2026 18:31:18 -0400 Subject: [PATCH 7/8] Refine discovery tests and reconcile documentation Keep feature tests in their original files while preserving meaningful coverage and removing an ineffective unauthenticated liveness case. Replace a timing-based concurrency assertion with an explicit request barrier and bring affected test names, fixtures, error checks, comments, and documentation into line with repository guidance. The specifications describe authenticated inventory ownership and verified provenance. Preserve public hardware displays, independent manual engine configuration, existing scheduler scoring, and the accepted worker-stdio freshness approximation. Suggested review order: 1. services/nvpair-node-scanner/spec.md, services/nvpair-node-info/spec.md, and services/nvpair-manual-nodes/spec.md as reference contracts established with the feature. 2. Scanner activity_test.go, cluster_membership_test.go, connreuse_test.go, refresh_test.go, telemetry_test.go, node_info_outage_test.go, daemon_test.go, directory_test.go, and enrich_hosts_test.go, then associated scanner comment changes: retained coverage and deterministic fixtures in their original files. 3. Node-info cluster_mtls_test.go, then main.go and README.md: transport fixtures and current public/protected inventory documentation. 4. Errors rename_test.go; proxy subscribed_test.go and proxy_test.go; engine-manager remote_test.go: retained consumer coverage and focused case names. 5. Remaining Go comment changes and scanner, manual-nodes, and TUI READMEs: authentication, ownership, display, and persistence behavior. Validation: all five affected Go module suites pass on Windows. Desktop type checks, lint, dead-code, service-contract verification, and unit tests pass (48 files, 281 tests, two existing platform skips, plus 26 VC runtime checks). gofmt and whitespace checks pass. Signed-off-by: Kaylee Lubick --- .../nvpair-cluster-manager/mdns_browser.go | 10 +- services/nvpair-engine-manager/remote_test.go | 6 +- services/nvpair-errors/rename_test.go | 21 +- services/nvpair-manual-nodes/README.md | 11 +- services/nvpair-node-info/README.md | 7 +- .../nvpair-node-info/cluster_mtls_test.go | 8 +- services/nvpair-node-info/main.go | 21 +- services/nvpair-node-scanner/README.md | 9 +- services/nvpair-node-scanner/activity_test.go | 42 +-- .../cluster_membership_test.go | 34 +- .../nvpair-node-scanner/connreuse_test.go | 4 +- services/nvpair-node-scanner/daemon.go | 98 ++---- services/nvpair-node-scanner/daemon_test.go | 308 +++++++++--------- .../nvpair-node-scanner/directory_test.go | 29 +- services/nvpair-node-scanner/enrich_hosts.go | 6 +- .../nvpair-node-scanner/enrich_hosts_test.go | 57 +++- .../node_info_outage_test.go | 8 +- services/nvpair-node-scanner/refresh_test.go | 128 +++----- services/nvpair-node-scanner/scanner.go | 7 +- .../nvpair-node-scanner/telemetry_test.go | 20 +- services/nvpair-proxy/proxy_test.go | 107 ++---- services/nvpair-proxy/subscribed_test.go | 28 +- services/nvpair-tui/README.md | 5 +- services/nvpair-tui/ui/nodesmodel.go | 8 +- services/nvpair-tui/ui/nodeswire.go | 14 +- services/nvpair-tui/ui/nodetelemetry.go | 8 +- services/nvpair-ui-broker/broker.go | 32 +- services/nvpair-ui-broker/discovery.go | 26 +- services/nvpair-ui-broker/nodeinfo.go | 11 +- .../nvpair-workload-manager/relaysource.go | 12 +- services/shared/clustertrust/mesh.go | 7 +- 31 files changed, 428 insertions(+), 664 deletions(-) diff --git a/services/nvpair-cluster-manager/mdns_browser.go b/services/nvpair-cluster-manager/mdns_browser.go index 1d2d1c32a..000064b67 100644 --- a/services/nvpair-cluster-manager/mdns_browser.go +++ b/services/nvpair-cluster-manager/mdns_browser.go @@ -23,7 +23,7 @@ import ( "nvpair-shared/noderec" ) -// Browser holds the relay-fed peer map, keyed by the directory hostUuid. +// Browser resolves relay records by discovery record ID and host UUID. type Browser struct { mu sync.RWMutex relay map[string]discovery.Node @@ -76,10 +76,10 @@ func (b *Browser) setRelay(nodes []noderec.DirectoryNode) { b.mu.Unlock() } -// Resolve returns a node's addresses in the order it ranked them, plus its -// cluster-manager port, for a UUID or nodeId (hostname); a UUID match wins. The -// list is what a caller dials down until one address answers — the publisher's -// ranking is evidence from its vantage point, not a promise about this host's. +// Resolve returns ranked addresses and the cluster-manager port. Exact record +// ID and host UUID lookups take precedence over hostname matching. Callers try +// the addresses in order because the publisher's ranking does not establish +// which addresses this host can reach. func (b *Browser) Resolve(idOrUUID string) (hosts []string, port int, ok bool) { b.mu.RLock() defer b.mu.RUnlock() diff --git a/services/nvpair-engine-manager/remote_test.go b/services/nvpair-engine-manager/remote_test.go index acb37f5f2..a2240621e 100644 --- a/services/nvpair-engine-manager/remote_test.go +++ b/services/nvpair-engine-manager/remote_test.go @@ -12,7 +12,7 @@ import ( "nvpair-shared/noderec" ) -func TestRunRemoteRequiresNode(t *testing.T) { +func TestRunRemote_RejectsMissingNode(t *testing.T) { var out bytes.Buffer m := NewManager(NewCodec(&out), &Executor{progress: newProgressHub()}, nil) id := json.RawMessage("1") @@ -21,7 +21,7 @@ func TestRunRemoteRequiresNode(t *testing.T) { mustContain(t, out.String(), "node is required") } -func TestRunRemoteUnknownPeer(t *testing.T) { +func TestRunRemote_RejectsUnknownPeer(t *testing.T) { var out bytes.Buffer m := NewManager(NewCodec(&out), &Executor{progress: newProgressHub()}, nil) id := json.RawMessage("1") @@ -30,7 +30,7 @@ func TestRunRemoteUnknownPeer(t *testing.T) { mustContain(t, out.String(), "not a discovered ec peer") } -func TestRunRemoteNotClustered(t *testing.T) { +func TestRunRemote_RejectsRequestsOutsideCluster(t *testing.T) { var out bytes.Buffer // Peer is discovered, but this node has no mesh (not clustered), so the // pinned dial can't be built. diff --git a/services/nvpair-errors/rename_test.go b/services/nvpair-errors/rename_test.go index 1f75a8347..ad9c85a01 100644 --- a/services/nvpair-errors/rename_test.go +++ b/services/nvpair-errors/rename_test.go @@ -4,8 +4,6 @@ package main import ( - "io" - "testing" "github.com/stretchr/testify/assert" @@ -14,18 +12,8 @@ import ( "nvpair-shared/noderec" ) -// nopReadWriter is a codec transport that reads EOF and discards writes, for -// tests that drive the manager's helpers directly. -type nopReadWriter struct{} - -func (nopReadWriter) Read([]byte) (int, error) { return 0, io.EOF } -func (nopReadWriter) Write(p []byte) (int, error) { return len(p), nil } - -// TestDirectoryToPeerKeysByHostUUID: a peer is keyed by its stable hostUuid (the -// same identity it stamps as the origin of its errors and uses as its own -// localNodeID), so the self-check and EvictNode survive a peer's PC rename and -// never conflate two same-named peers. Host stays the hostname for dialing / -// display. +// Error origin, peer eviction, and local self-checks share a stable host UUID. +// Keeping the hostname separate prevents renames from changing that identity. func TestDirectoryToPeer_KeysByHostUUID(t *testing.T) { const hostUUID = "uuid-x" const hostName = "host-x" @@ -49,10 +37,9 @@ func TestDirectoryToPeer_KeysByHostUUID(t *testing.T) { assert.Equal(t, []string{primaryAddress, secondaryAddress}, p.Addresses, "peer addresses") } -// TestSetLocalNodeID: the broker's --node-id override replaces the hostname -// default so local errors are attributed to this node's stable UUID. +// The broker's --node-id override attributes local errors to a stable UUID. func TestSetLocalNodeID_RekeysLocalErrors(t *testing.T) { - m := NewManager(NewCodec(nopReadWriter{})) + m := NewManager(NewCodec(newCaptureRW())) m.SetLocalNodeID("uuid-self") assert.Equal(t, "uuid-self", m.LocalNodeID()) // An empty override must not blank a known id. diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index 6874d7b50..ade1d5088 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -88,7 +88,11 @@ Emitted so the supervising broker can forward them into the `nvpair-errors` pipe Add a node by address. The manager immediately probes it and emits a `node/discovered` event. -A hostname is preferred over an IP literal: probe clients disable keep-alives specifically so every probe re-resolves the name, which lets a node that gets a new address recover on its own. An IP-literal entry is dead once the device is renumbered. Supply the address on its own — a `host:port` string is not parsed, because ports are appended to it, so such an entry reads permanently down. +A hostname is preferred over an IP literal: all probes open fresh connections +so they re-resolve the name after an address change, including authenticated +hardware probes. An IP-literal entry cannot +follow a renumbered device. Supply the address on its own — a `host:port` string +is not parsed, because ports are appended to it. ```json {"jsonrpc":"2.0","id":1,"method":"node/add","params":{"address":"10.0.1.50","name":"my-server"}} @@ -99,7 +103,7 @@ A hostname is preferred over an IP literal: probe clients disable keep-alives sp | `address` | Yes | IP address or hostname of the node, with no port | | `name` | No | Friendly name (used as node ID; defaults to `manual:
`) | | `tls_port` | No | Override the node-info port (default `14318`). Hardware reads always require pinned mTLS. Echoed back as `tls_enabled` | -| `mtls` | No | Stored and echoed back as `mtls_required`. The probe transport itself is chosen by `tls_port` and live cluster membership, so this field records intent rather than driving it | +| `mtls` | No | Configuration hint echoed as `mtls_required` when `tls_port` is set. Hardware probes always use mTLS with current cluster pins | Response: the initial node status object. @@ -146,7 +150,8 @@ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, fo A node can have any combination of these, or none if the target is unreachable. Hardware reads require exact current cluster pins and matching `hostUuid` and `clusterUuid` in the response. They use pooled mTLS clients after identifying -the peer, refuse redirects, and never fall back to public HTTP or operator CAs. +the peer, close each connection after the response, refuse redirects, and never +fall back to public HTTP or operator CAs. Unverified configurations report `identityVerification: "unverified"` and supply no service measurements. Explicitly configured engine routes remain available. Desktop and TUI may independently read public HTTP for hardware display. diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index fb86f2479..3df16c883 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -16,8 +16,9 @@ Two surfaces: It does **not** advertise itself over mDNS. Discovery is centralized in the `nvpair-node-scanner` daemon: the broker registers this service's `ni` port on -the node's one `_nvpair-node` record. Remote scanners currently read its public -HTTP inventory; the owned local scanner receives private IPC snapshots. +the node's one `_nvpair-node` record. Remote services read inventory over pinned +mTLS; the owned local scanner receives private IPC snapshots. Desktop and TUI +continue to read public HTTP hardware data for display only. ### Private inventory IPC @@ -118,7 +119,7 @@ Field notes: - `telemetryValid` and `msSince` describe the node-wide GPU utilization snapshot. `telemetryValid` is `true` after the collector has produced a usable GPU sample; `msSince` is that sample's age in milliseconds at response time. A failed collection retains the last usable sample and lets its age increase. Before the first usable sample, and on platforms without dynamic GPU telemetry, the response reports `telemetryValid:false` and `msSince:0`; consumers must ignore the age while validity is false. - `clusterUuid` is the cluster principal this node currently holds. It has three distinct states on the wire: **absent** means unknown, **present and empty** means this node belongs to no cluster, and a value is that principal. A consumer must not read absent as unclustered — that is how a node too old to report the field answers, and also how this node answers before its parent has told it anything, so acting on it would clear a correct annotation elsewhere in the fleet. - With `--cluster-dir`, including under the broker, `clusterUuid` comes from live trust-store membership. Without a cluster directory, a parent may push it over stdin using `nodeinfo:set-cluster-identity`; the field stays absent until that first push. These are separate deployment modes. -- `clusterUuid` exists so a peer can learn this node's membership without its mDNS record. Membership otherwise travels only as the `cluster-uuid=` TXT key, which a consumer reads once per record *change*; a consumer that misses that change keeps the previous value indefinitely, and one still holding a departed node's principal will suppress the invite that would bring it back. +- `clusterUuid` is an identity claim. Public HTTP cannot establish membership or change a verified node's identity; service consumers require the protected response and a matching exact certificate pin. - All dynamic fields and the `cpu` / `memory` objects use `omitempty`: a value the service couldn't read is dropped from the JSON entirely rather than reported as a misleading literal zero. A genuinely idle CPU renders the same as "unknown" — that ambiguity is intentional and benign. - `vram_bytes` is reported through DXGI on Windows, `nvidia-smi` on Linux, and IORegistry on macOS. On a unified-memory NVIDIA GPU such as DGX Spark, Linux uses total physical system memory for `vram_bytes` and the independently sampled system-memory usage for `vram_used_bytes`. On Apple Silicon, `vram_bytes` is total physical unified memory and `vram_used_bytes` is the GPU driver's mapped allocation (`Alloc system memory`), not whole-system RAM usage or the momentarily active subset. diff --git a/services/nvpair-node-info/cluster_mtls_test.go b/services/nvpair-node-info/cluster_mtls_test.go index e104f9ee5..4cf154ff3 100644 --- a/services/nvpair-node-info/cluster_mtls_test.go +++ b/services/nvpair-node-info/cluster_mtls_test.go @@ -32,7 +32,7 @@ import ( func genLeaf(t *testing.T, uuid string) (certPEM, keyPEM []byte) { t.Helper() pub, priv, err := ed25519.GenerateKey(rand.Reader) - require.NoError(t, err, "genkey") + require.NoError(t, err) serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) require.NoError(t, err) uri, err := url.Parse("urn:nvpair:node:" + uuid) @@ -65,8 +65,12 @@ func setupNode(t *testing.T, certPEM, keyPEM []byte, pins map[string][]byte) str require.NoError(t, os.WriteFile(filepath.Join(dir, "node.key"), keyPEM, 0o600)) td := filepath.Join(dir, "trusted") require.NoError(t, os.MkdirAll(td, 0o700)) + type peerPin struct { + NodeUUID string `json:"nodeUuid"` + CertPEM string `json:"certPem"` + } for uuid, pcert := range pins { - body, err := json.Marshal(map[string]string{"nodeUuid": uuid, "certPem": string(pcert)}) + body, err := json.Marshal(peerPin{NodeUUID: uuid, CertPEM: string(pcert)}) require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(td, uuid+".json"), body, 0o600)) } diff --git a/services/nvpair-node-info/main.go b/services/nvpair-node-info/main.go index 644954900..b188b063e 100644 --- a/services/nvpair-node-info/main.go +++ b/services/nvpair-node-info/main.go @@ -79,26 +79,19 @@ type NodeInfoResponse struct { TelemetryValid bool `json:"telemetryValid"` MSSince int64 `json:"msSince"` // HostUUID is this node's stable per-host identity (the same value the - // node-scanner advertises as uuid= and the cluster uses as nodeUuid). It lets - // a consumer that reaches this node only over HTTP — notably a user-added - // manual node, which never sees this host's mDNS record — key it by its - // permanent identity instead of the address it was typed under. + // node-scanner advertises as uuid= and the cluster uses as nodeUuid). + // Services authenticate it against the serving peer's pinned certificate; + // a public HTTP response cannot establish or replace a node's identity. HostUUID string `json:"hostUuid,omitempty"` // ClusterUUID is the cluster principal this node currently holds, empty when // it belongs to no cluster, and absent when this node does not (yet) know. It - // is the mDNS-independent way for a peer to learn this node's membership: - // membership otherwise travels only as the cluster-uuid= TXT key on this - // host's mDNS record, so a peer that stops receiving that record keeps its - // last observed value forever — and a peer still carrying a departed node's - // principal suppresses the invite that would bring it back. + // accompanies authenticated inventory. Membership changes come from cluster + // state and verified discovery; public HTTP responses cannot change them. // // A pointer with omitempty, so all three states are distinct on the wire: // absent means unknown, present-and-empty means unclustered, present means - // that principal. A consumer must not read absent as unclustered — that is - // how a node too old to report the field answers, and also how this node - // answers before its parent has told it anything. Claiming "no cluster" on no - // evidence would have a peer clear a correct annotation and offer an invite - // its target will reject. + // that principal. Absence supplies no membership information, including + // before a launch without --cluster-dir receives its first broker update. ClusterUUID *string `json:"clusterUuid,omitempty"` } diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index 3921a5ded..19477e148 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -29,6 +29,8 @@ Concretely, the daemon: engine-manager over loopback; local hardware comes from private worker IPC. Local registry publications use retained models and wake the model sweep, so an HTTP read cannot hold the lock needed by owned hardware notifications. + Queued and in-flight model results are discarded if the peer's authenticated + address list changes, including a secondary address used for failover. - **Samples scheduling telemetry** from healthy nodes every two seconds with stable per-node jitter. Consecutive remote failures back off to 4, 8, 16, then 30 seconds and reset on success. Local inventory and samples arrive every two @@ -60,9 +62,6 @@ Concretely, the daemon: response-size limits, address failover, and one connection per node-info origin. Requests never follow redirects to another endpoint or transport. -Queued and in-flight model results are discarded if the peer's authenticated -address list changes, including a secondary address used for failover. - ## Communication Bidirectional newline-delimited JSON-RPC 2.0. Stdio by default; `--ipc ` switches to a Unix domain socket or Windows named pipe. @@ -116,7 +115,7 @@ Emitted as the directory changes. Each carries the affected `DirectoryNode` (ide Records also carry `discoveryRecordId` and `identityVerification`. The local registry record is `local`; browsed records are `unverified`, including when their claimed -principal has a pin. A successful read creates a distinct `peer:` record +principal has a pin. A successful read creates a distinct opaque peer record with `pinned` verification and `trusted: true`. Failed discoveries cannot alter that record. Discovery record IDs identify candidates independently of UUID and principal claims; address order does not affect the ID. @@ -126,6 +125,8 @@ principal claims; address order does not affect the ID. - `discovery:node-removed` — a node aged out (identity + last-known fields) - `discovery:node-telemetry` — compact maximum-GPU utilization, validity, and sample age for broker-internal scheduling +- `discovery:node-info` — a full authenticated hardware snapshot with its + discovery record ID and identity verification, forwarded through the broker A one-shot `ready` notification is sent on startup carrying the daemon version. The broker replays its service registrations on every scanner spawn, so a restarted daemon is repopulated without needing to compare an epoch. diff --git a/services/nvpair-node-scanner/activity_test.go b/services/nvpair-node-scanner/activity_test.go index bf3301fff..9d80b733e 100644 --- a/services/nvpair-node-scanner/activity_test.go +++ b/services/nvpair-node-scanner/activity_test.go @@ -5,44 +5,21 @@ package main import ( "encoding/json" - "fmt" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "nvpair-shared/noderec" ) -// The probes this file is about are the ones that fail on a node BECAUSE it is -// working: a machine saturated by its own inference load has no CPU left to -// answer a node-info GET or accept a TCP connect within a second, so the -// eviction path reads it as gone and drops it mid-request. The proxies, meanwhile, -// are receiving that node's response bytes. These tests pin that evidence beating -// the probe. - -// activityDaemon is the minimum daemon the activity path needs. It deliberately -// has no HTTP client and no registry: the point of every test here is what -// happens when nothing answers. +// activityDaemon initializes only the state used to record inference activity. func activityDaemon() *daemon { return &daemon{lastActivityAt: make(map[string]time.Time)} } -// unreachableNode advertises a node-info port nothing is listening on, so the -// TCP sweep in reachable is guaranteed to fail. -func unreachableNode(t *testing.T, hostUUID string) RawNode { - t.Helper() - port := closedPort(t) - return RawNode{ - ID: hostUUID, - Addresses: []string{"127.0.0.1"}, - TXT: []string{"v=1", "uuid=" + hostUUID, "ip=127.0.0.1", fmt.Sprintf("ni=%d", port)}, - } -} - -// TestFreshActivityKeepsANodeNoProbeCanReach is the fix. Without it a node that -// is streaming a generation back to us is evicted for failing a probe it was too -// busy to answer. +// An authenticated peer may be too busy serving inference to answer node-info. func TestFreshActivity_KeepsANodeNoProbeCanReach(t *testing.T) { f := newDiscoveryPeerFixture(t, "busy-node") f.discover(t) @@ -56,23 +33,14 @@ func TestFreshActivity_KeepsANodeNoProbeCanReach(t *testing.T) { require.True(t, d.reachable(n), "a node that just returned inference bytes must be kept despite an unanswerable probe") } -// Evidence has to expire, or a node that has genuinely gone would be held by a -// report from an hour ago and never age out. -func TestStaleActivity_DoesNotKeepANode(t *testing.T) { - d := activityDaemon() - n := unreachableNode(t, "departed-node") - - d.noteActivity("departed-node", (activityFreshness + time.Second).Milliseconds()) - require.False(t, d.reachable(n), "activity older than activityFreshness must not keep an unreachable node") -} - // Activity is credited per node. One busy peer vouching for itself must not // vouch for a different peer that really has left. func TestActivity_IsCreditedPerNode(t *testing.T) { d := activityDaemon() d.noteActivity("busy-node", 0) - require.False(t, d.reachable(unreachableNode(t, "other-node")), "one node's activity must not keep a different node alive") + _, ok := d.activitySince("other-node") + assert.False(t, ok, "one node's report must not record activity for another") } // The report carries an age rather than a timestamp so the two processes need no diff --git a/services/nvpair-node-scanner/cluster_membership_test.go b/services/nvpair-node-scanner/cluster_membership_test.go index f7105a257..28188d20a 100644 --- a/services/nvpair-node-scanner/cluster_membership_test.go +++ b/services/nvpair-node-scanner/cluster_membership_test.go @@ -33,11 +33,8 @@ func newMembershipTestDaemon(clusterDir string) *daemon { } } -// TestModelFetchTransportFollowsMembership pins the transport the daemon uses to -// read a model inventory. A peer's inventory is cluster data, so it is fetched -// only over mTLS pinned to that peer's principal; THIS node's own engine-manager -// is read over loopback in plaintext, which is what keeps a standalone machine's -// own model list working while it belongs to no cluster. +// Remote model inventories require pinned mTLS. The owned engine-manager keeps +// its loopback HTTP path before and after joining a cluster. func TestModelFetchTransport_FollowsMembership(t *testing.T) { clusterDir := filepath.Join(t.TempDir(), "cluster") d := newMembershipTestDaemon(clusterDir) @@ -68,9 +65,7 @@ func TestModelFetchTransport_FollowsMembership(t *testing.T) { assert.Equal(t, "http", scheme) } -// TestClusterUUIDGatedOnLiveMembership pins the invite-deadlock fix at the -// scanner: cluster-uuid= is advertised only while actually a member, not because -// a keypair from a past membership is still on disk. +// A retained keypair alone must not advertise active cluster membership. func TestClusterUUID_ReflectsLiveMembership(t *testing.T) { // A left/removed node: keypair present, but no active admission and no pins. // It must NOT advertise a cluster-uuid= (so peers see it as invitable). @@ -90,13 +85,7 @@ func TestClusterUUID_ReflectsLiveMembership(t *testing.T) { require.Equal(t, "principal-B", d2.reg.record().ClusterUUID, "clustered node advertised cluster-uuid") } -// TestClusterUUIDAppearsOnJoinWithoutRestart is the advertisement half of the -// half-clustered regression guard. A scanner that started before this node had -// any cluster identity must begin advertising cluster-uuid= once the join lands -// on disk. Peers key their pins on that TXT value, so a scanner that could only -// learn it at startup left this node looking unclustered to the whole fleet: -// skipped by every peer's workload broadcast and dropped as a routing target, -// while still sitting in the roster. +// A running scanner must advertise admission changes without a process restart. func TestClusterUUID_AppearsOnJoinWithoutRestart(t *testing.T) { clusterDir := filepath.Join(t.TempDir(), "cluster") d := newMembershipTestDaemon(clusterDir) @@ -112,9 +101,7 @@ func TestClusterUUID_AppearsOnJoinWithoutRestart(t *testing.T) { require.Equal(t, "principal-C", d.reg.record().ClusterUUID, "post-join cluster-uuid") } -// TestReconcileIdentityFollowsMembership covers the identity half of the same -// pass: it re-advertises only when live membership disagrees with what the -// registry is publishing, so the common case costs a string compare. +// Membership reconciliation updates the local registry after joining a cluster. func TestReconcileIdentity_FollowsMembership(t *testing.T) { clusterDir := filepath.Join(t.TempDir(), "cluster") d := newMembershipTestDaemon(clusterDir) @@ -128,15 +115,8 @@ func TestReconcileIdentity_FollowsMembership(t *testing.T) { require.Equal(t, "principal-self", d.reg.record().ClusterUUID, "post-join cluster-uuid") } -// TestPeerLeavingItsClusterClearsInviteGate pins the directory half of the -// reported failure: a machine kept showing peers as belonging to another -// cluster, with invites disabled, after every one of them had left. -// -// A client suppresses an invite a clustered peer would reject, and it decides -// that from this annotation, which is only ever derived from the peer's own -// record. So an entry that keeps a cluster-uuid= the peer no longer advertises -// blocks a legitimate invite for as long as the entry survives — the annotation -// has to follow the record down as readily as it followed it up. +// An unverified announcement's membership claim is display data. Clearing it +// must make the row available for pairing again. func TestPeerLeavingItsCluster_ClearsInviteGate(t *testing.T) { d := newMembershipTestDaemon(filepath.Join(t.TempDir(), "cluster")) const peerUUID = "peer-uuid" diff --git a/services/nvpair-node-scanner/connreuse_test.go b/services/nvpair-node-scanner/connreuse_test.go index ed58e86af..02b678a6c 100644 --- a/services/nvpair-node-scanner/connreuse_test.go +++ b/services/nvpair-node-scanner/connreuse_test.go @@ -21,9 +21,7 @@ import ( "nvpair-shared/clustertrusttest" ) -// TestModelsClient_ReusesPeerConnections is the regression for the 15s -// enrichment handshake storm: modelsClient used to build a fresh Transport -// per peer fetch. Several /v1/models GETs must ride ONE connection. +// Repeated model inventory reads reuse one authenticated peer connection. func TestModelsClient_ReusesPeerConnections(t *testing.T) { selfDir := t.TempDir() peerDir := t.TempDir() diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 9a6dedf54..23268d138 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -30,16 +30,9 @@ import ( "nvpair-shared/reach" ) -// daemon is the promoted node-scanner: it advertises this -// node as ONE _nvpair-node record built from a registry of relayed service -// registrations, browses _nvpair-node to build a directory of every LAN node, and -// serves the discovery:* RPC (register/unregister/update-txt/get-nodes/ -// reload-identity) while emitting discovery:node-* events. -// -// Post-cutover this is the sole discovery path: the legacy _nvpair-node-info -// browse and node/* stream are gone, and every consumer registers/subscribes -// through the broker relay, which the broker builds from this daemon's -// discovery:node-* stream and fans out to consumers as discovery:nodes snapshots. +// daemon owns local registration, candidate discovery, and authenticated peer +// inventory. It publishes discovery:node-* events for the broker relay, which +// delivers directory snapshots to service consumers. type daemon struct { codec *Codec mesh *clustertrust.Mesh @@ -51,11 +44,8 @@ type daemon struct { // registry-driven self directory entry (publishSelf), which is anchored to // our own identity rather than to hearing our own multicast. instance string - // modelsHTTP fetches the local /v1/models. It gets a longer timeout than node-info - // (node-info): engine-manager's models sweep is bounded by its own, larger - // deadline, so a peer with several engines can legitimately take longer than - // a node-info probe. A shorter client would give up mid-sweep and silently - // drop that peer's models. + // modelsHTTP reads the owned engine-manager over loopback. Model sweeps have + // a longer timeout than node-info because they may inspect several engines. modelsHTTP *http.Client // modelsClients is the long-lived per-peer mTLS pool for LAN /v1/models // fetches. A throwaway Transport every 15s paid a full handshake each time. @@ -766,16 +756,9 @@ func (d *daemon) reloadTrust() { d.wakeAuthentication() } -// reconcileIdentity re-advertises this node when live membership no longer -// matches the cluster principal the registry is publishing — the join or leave -// that reloadIdentity exists to pick up. Peers key their pins on our -// cluster-uuid= TXT, so a node that became a member must start advertising it -// without a restart; nothing else tells the fleet. -// -// The comparison is against the registry's current value rather than a change -// signal, so it is safe to call on any event: it is an in-memory string compare, -// and reloadIdentity's file read and interface enumeration only happen when the -// answer actually differs. +// reconcileIdentity republishes local identity when current membership differs +// from the registry. Announcements expose membership for pairing displays; +// service trust still requires an authenticated peer inventory response. func (d *daemon) reconcileIdentity() { want := "" if d.mesh.Clustered() { @@ -850,30 +833,10 @@ func (d *daemon) emitNodeInfo(node noderec.DirectoryNode, info noderec.NodeInfo) } } -// reloadIdentity re-resolves this node's advertised identity (hostUuid, -// clusterUuid, ip) and re-advertises if any of it changed. It has three triggers: -// -// - Membership change: the mesh watch calls this when this node joins or leaves -// a cluster, so cluster-uuid= appears in (or disappears from) this node's mDNS -// record within one refresh interval. This is what lets peers key their pins -// on us: without the TXT key a peer treats us as unclustered, skips us in its -// workload broadcast, and drops us as a routing target — while we remain in -// its roster. It must not depend on a process restart. -// - Fresh-host startup order: the scanner spawns before cluster-manager, so it -// mints its own node-id.json and advertises that uuid= before cluster-manager -// writes identity.json. Once cluster-manager is up (the broker triggers a -// reload), the re-resolve picks up identity.json so uuid= converges on the -// cluster principal. -// - Network change: watchAddress calls this on a netmon event so the advertised -// addresses track the host's live network instead of freezing at the startup -// value — matching how the responder keeps its A records live. -// -// Address evidence that arrives later without a network change is handled by -// refreshAdvertisedAddressesOnce, which re-ranks without re-reading identity. -// -// Safe to call concurrently: selfMu serializes it against publishSelf, and -// setIdentity/UpdateTXT are each mutex-guarded and idempotent for unchanged -// fields. +// reloadIdentity resolves this host's current identity, membership, and addresses +// after cluster or network changes. It removes a superseded local UUID and +// republishes the registry without relying on a process restart. +// selfMu serializes it with registry publications and owned hardware snapshots. func (d *daemon) reloadIdentity() { d.selfMu.Lock() defer d.selfMu.Unlock() @@ -950,9 +913,8 @@ func (d *daemon) observedAddresses() map[string]bool { return maps.Clone(d.observed) } -// peerAddresses returns every address known for other nodes, for the on-link -// inference in localAddresses. Self is excluded: our own record loops back -// through mDNS, and seeing our own address proves nothing about the link. +// peerAddresses returns authenticated remote addresses for the on-link evidence +// used by localAddresses. Local and unverified discovery records contribute none. func (d *daemon) peerAddresses() []string { self := "" if d.reg != nil { @@ -1500,13 +1462,10 @@ func (d *daemon) expireRetainedPeer(target noderec.DirectoryNode) { d.emit(noderec.NotifyNodeRemoved, peer) } -// refreshModelsOnce snapshots every directory node advertising engine-manager -// (em) with a usable IP and refreshes each concurrently, bounded by -// modelsRefreshConcurrency so an unexpectedly large network can't spawn an -// unbounded burst of fetches. Running the sweep to completion (wg.Wait) before -// the loop reads the next tick makes overlapping sweeps impossible, and each -// per-node goroutine uses modelsHTTP's own timeout, so one slow peer never -// consumes another's budget. +// refreshModelsOnce refreshes local and authenticated peer model inventories. +// Each sweep waits for its bounded workers, preventing overlapping sweeps. +// Requests have independent timeouts, so one slow peer does not consume another's +// request budget. func (d *daemon) refreshModelsOnce() { sem := make(chan struct{}, modelsRefreshConcurrency) var wg sync.WaitGroup @@ -1527,22 +1486,11 @@ func (d *daemon) refreshModelsOnce() { wg.Wait() } -// refreshNodeModels fetches one peer's /v1/models and, on success, applies it to -// the directory under the removed/re-addressed guard (directory.applyModels). -// A failed fetch retains the last successful inventory: no cache write, no -// directory change, no event. On a change it refreshes the enrichment caches -// (so a later browse-driven fallback can't revert the directory to a stale -// value) and emits exactly one discovery:node-updated carrying the full node. -// Returns whether it emitted an update (used by tests). -// -// The fetch dials our OWN engine-manager over loopback, never the advertised -// LAN ip=: this loop — not the registry-driven publishSelf — is what converges -// model-only changes (a pull/delete, a late engine start) that don't move the -// registry, so on the Windows self-LAN firewall block this daemon guards -// against, a LAN dial would make self's model list freeze at its -// startup value. The stored ip is still passed to applyModels as the -// stale-address guard, so an in-flight result can't overwrite a re-addressed -// node. Peers are still dialed at their advertised address. +// refreshNodeModels snapshots the complete address list before fetching model +// inventory. The shared result path validates current endpoints and remote proof +// before changing the directory, caches, or notifications. Failed reads preserve +// the last accepted inventory. Local reads use the owned engine-manager's +// loopback HTTP endpoint; remote reads require the verified peer's pinned mTLS. func (d *daemon) refreshNodeModels(hostUUID, ip string, emPort int, clusterUUID string) bool { node, present := d.dir.get(hostUUID) if !present { diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index 7832fbc83..9c12ca511 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -982,6 +982,141 @@ func TestDiscovery_DiscardsModelResponseAfterSavedCertificateReplacement(t *test assert.Equal(t, response.LoadedByEngine, after.LoadedByEngine) } +func TestRefreshNodeModels_AcceptsPinnedPeersWithMultipleAddresses(t *testing.T) { + const peerUUID = "peer-uuid" + f := newDiscoveryPeerFixture(t, peerUUID) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + f.info.Peer.Addresses = []string{"127.0.0.1", "127.0.0.2"} + type modelResponse struct { + Models []string `json:"models"` + } + response := modelResponse{Models: []string{"reported-model"}} + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(response)) + return true + } + f.discover(t) + + assert.True(t, f.daemon.refreshNodeModels(peerUUID, "127.0.0.1", port, peerUUID)) + + node, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + assert.Equal(t, response.Models, node.Models) +} + +func TestDiscovery_DiscardsModelsAfterSecondaryAddressChanges(t *testing.T) { + const peerUUID = "peer-uuid" + const primaryAddress = "127.0.0.2" + + test := func(name, secondaryAddress string, wantAccepted bool) { + t.Run(name, func(t *testing.T) { + f := newDiscoveryPeerFixture(t, peerUUID) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + // Only the secondary address reaches this peer's server. + f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.1"} + type modelResponse struct { + Models []string `json:"models"` + } + response := modelResponse{Models: []string{"reported-model"}} + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(response)) + return true + } + f.discover(t) + original, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + client, ok := f.daemon.modelsClients.Client(peerUUID) + require.True(t, ok) + closed, release := make(chan struct{}), make(chan struct{}) + client.Transport = heldModelResponseTransport{RoundTripper: client.Transport, closed: closed, release: release} + changed := make(chan bool, 1) + done := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) + go func() { + defer close(done) + changed <- f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID) + }() + select { + case <-closed: + case <-time.After(8 * time.Second): + require.FailNow(t, "model response did not reach the point before directory update") + } + + // Refresh authoritative inventory while the old model result waits to + // be applied. The primary address, port, and pin remain unchanged. + f.info.Peer.Addresses = []string{primaryAddress, secondaryAddress} + require.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), peerUUID, original.CandidateIPs(), original.Services[noderec.ServiceNodeInfo].Port)) + before, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + require.Equal(t, []string{primaryAddress, secondaryAddress}, before.CandidateIPs()) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + releaseOnce.Do(func() { close(release) }) + <-done + + assert.Equal(t, wantAccepted, <-changed) + after, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + if wantAccepted { + assert.Equal(t, response.Models, after.Models) + return + } + assert.Equal(t, before, after) + assert.Empty(t, output.Bytes()) + }) + } + + test("unchanged addresses accept the model response", "127.0.0.1", true) + test("removed secondary address discards the model response", "127.0.0.3", false) +} + +func TestDiscovery_DiscardsQueuedModelsForChangedAddresses(t *testing.T) { + const peerUUID = "peer-uuid" + const primaryAddress = "127.0.0.2" + f := newDiscoveryPeerFixture(t, peerUUID) + port := portFromURL(t, f.server.URL) + f.info.Peer.Services[noderec.ServiceEngineManager] = port + f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.1"} + type modelResponse struct { + Models []string `json:"models"` + } + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(modelResponse{Models: []string{"obsolete-model"}})) + return true + } + f.discover(t) + original, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + + // The model sweep queued the old addresses before receiving new inventory. + f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.3"} + require.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), peerUUID, original.CandidateIPs(), original.Services[noderec.ServiceNodeInfo].Port)) + before, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + assert.False(t, f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID)) + + after, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + assert.Equal(t, before, after) + assert.Empty(t, output.Bytes()) +} + func TestDiscovery_PeerActivityDoesNotKeepSpoofedDiscoveryReachable(t *testing.T) { f := newDiscoveryPeerFixture(t, "peer-uuid") f.discover(t) @@ -1000,6 +1135,25 @@ func TestDiscovery_PeerActivityDoesNotKeepSpoofedDiscoveryReachable(t *testing.T assert.False(t, f.daemon.reachable(f.raw), "activity must not retain a peer after its certificate is removed") } +func TestPeerReachable_RejectsCachedLivenessAfterReadmission(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + f.daemon.noteActivity("peer-uuid", 0) + require.True(t, f.daemon.peerReachable("peer-uuid")) + require.True(t, f.daemon.reachable(f.raw)) + + // Readmission changes the proof binding even when both certificates remain. + // Availability checks must reject old evidence before a reload notification. + clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) + + assert.False(t, f.daemon.peerReachable("peer-uuid")) + assert.False(t, f.daemon.reachable(f.raw)) + + f.daemon.reloadTrust() + f.daemon.refreshCandidatesOnce(context.Background()) + assert.True(t, f.daemon.reachable(f.raw)) +} + func TestDiscovery_RefreshesOnlyAuthenticatedInventory(t *testing.T) { const updatedPort = 11435 f := newDiscoveryPeerFixture(t, "peer-uuid") @@ -1484,157 +1638,3 @@ func TestDiscovery_FailedAnnouncementUpdateStillAllowsRemovalOfUnavailablePeer(t _, present := f.daemon.dir.get("peer-uuid") assert.False(t, present) } - -func TestRefreshNodeModels_AcceptsPinnedPeersWithMultipleAddresses(t *testing.T) { - const peerUUID = "peer-uuid" - f := newDiscoveryPeerFixture(t, peerUUID) - port := portFromURL(t, f.server.URL) - f.info.Peer.Services[noderec.ServiceEngineManager] = port - f.info.Peer.Addresses = []string{"127.0.0.1", "127.0.0.2"} - type modelResponse struct { - Models []string `json:"models"` - } - response := modelResponse{Models: []string{"reported-model"}} - f.respond = func(w http.ResponseWriter, r *http.Request) bool { - if r.URL.Path != "/v1/models" { - return false - } - assert.NoError(t, json.NewEncoder(w).Encode(response)) - return true - } - f.discover(t) - - assert.True(t, f.daemon.refreshNodeModels(peerUUID, "127.0.0.1", port, peerUUID)) - - node, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - assert.Equal(t, response.Models, node.Models) -} - -func TestDiscovery_DiscardsModelsAfterSecondaryAddressChanges(t *testing.T) { - const peerUUID = "peer-uuid" - const primaryAddress = "127.0.0.2" - - test := func(name, secondaryAddress string, wantAccepted bool) { - t.Run(name, func(t *testing.T) { - f := newDiscoveryPeerFixture(t, peerUUID) - port := portFromURL(t, f.server.URL) - f.info.Peer.Services[noderec.ServiceEngineManager] = port - // Only the secondary address reaches this peer's server. - f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.1"} - type modelResponse struct { - Models []string `json:"models"` - } - response := modelResponse{Models: []string{"reported-model"}} - f.respond = func(w http.ResponseWriter, r *http.Request) bool { - if r.URL.Path != "/v1/models" { - return false - } - assert.NoError(t, json.NewEncoder(w).Encode(response)) - return true - } - f.discover(t) - original, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - client, ok := f.daemon.modelsClients.Client(peerUUID) - require.True(t, ok) - closed, release := make(chan struct{}), make(chan struct{}) - client.Transport = heldModelResponseTransport{RoundTripper: client.Transport, closed: closed, release: release} - changed := make(chan bool, 1) - done := make(chan struct{}) - var releaseOnce sync.Once - t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) - go func() { - defer close(done) - changed <- f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID) - }() - select { - case <-closed: - case <-time.After(8 * time.Second): - require.FailNow(t, "model response did not reach the point before directory update") - } - - // Refresh authoritative inventory while the old model result waits to - // be applied. The primary address, port, and pin remain unchanged. - f.info.Peer.Addresses = []string{primaryAddress, secondaryAddress} - require.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), peerUUID, original.CandidateIPs(), original.Services[noderec.ServiceNodeInfo].Port)) - before, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - require.Equal(t, []string{primaryAddress, secondaryAddress}, before.CandidateIPs()) - var output bytes.Buffer - f.daemon.codec = NewCodec(&output) - - releaseOnce.Do(func() { close(release) }) - <-done - - assert.Equal(t, wantAccepted, <-changed) - after, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - if wantAccepted { - assert.Equal(t, response.Models, after.Models) - return - } - assert.Equal(t, before, after) - assert.Empty(t, output.Bytes()) - }) - } - - test("unchanged addresses accept the model response", "127.0.0.1", true) - test("removed secondary address discards the model response", "127.0.0.3", false) -} - -func TestDiscovery_DiscardsQueuedModelsForChangedAddresses(t *testing.T) { - const peerUUID = "peer-uuid" - const primaryAddress = "127.0.0.2" - f := newDiscoveryPeerFixture(t, peerUUID) - port := portFromURL(t, f.server.URL) - f.info.Peer.Services[noderec.ServiceEngineManager] = port - f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.1"} - type modelResponse struct { - Models []string `json:"models"` - } - f.respond = func(w http.ResponseWriter, r *http.Request) bool { - if r.URL.Path != "/v1/models" { - return false - } - assert.NoError(t, json.NewEncoder(w).Encode(modelResponse{Models: []string{"obsolete-model"}})) - return true - } - f.discover(t) - original, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - - // The model sweep queued the old addresses before receiving new inventory. - f.info.Peer.Addresses = []string{primaryAddress, "127.0.0.3"} - require.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), peerUUID, original.CandidateIPs(), original.Services[noderec.ServiceNodeInfo].Port)) - before, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - var output bytes.Buffer - f.daemon.codec = NewCodec(&output) - - assert.False(t, f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID)) - - after, present := f.daemon.dir.get(peerUUID) - require.True(t, present) - assert.Equal(t, before, after) - assert.Empty(t, output.Bytes()) -} - -func TestPeerReachable_RejectsCachedLivenessAfterReadmission(t *testing.T) { - f := newDiscoveryPeerFixture(t, "peer-uuid") - f.discover(t) - f.daemon.noteActivity("peer-uuid", 0) - require.True(t, f.daemon.peerReachable("peer-uuid")) - require.True(t, f.daemon.reachable(f.raw)) - - // Readmission changes the proof binding even when both certificates remain. - // Availability checks must reject old evidence before a reload notification. - clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) - - assert.False(t, f.daemon.peerReachable("peer-uuid")) - assert.False(t, f.daemon.reachable(f.raw)) - - f.daemon.reloadTrust() - f.daemon.refreshCandidatesOnce(context.Background()) - assert.True(t, f.daemon.reachable(f.raw)) -} diff --git a/services/nvpair-node-scanner/directory_test.go b/services/nvpair-node-scanner/directory_test.go index 745a09c31..f777d4ebe 100644 --- a/services/nvpair-node-scanner/directory_test.go +++ b/services/nvpair-node-scanner/directory_test.go @@ -86,11 +86,7 @@ func TestToDirectoryNode_ProjectsClaimsWithoutGrantingTrust(t *testing.T) { assert.Equal(t, 11434, n.Services[noderec.ServiceOllama].Port) } -// TestToDirectoryNodeIgnoresSRVPort asserts the locked-decision contract: the -// SRV port on a browsed _nvpair-node record is a fixed, NON-authoritative -// constant that consumers MUST ignore — every service port comes only from the -// TXT map. Here the record's SRV port is a bogus value while TXT says ni=14318, -// and the directory entry must report the TXT port, never the SRV one. +// Candidate service ports come from TXT claims, independent of the SRV port. func TestToDirectoryNode_IgnoresSRVPort(t *testing.T) { raw := RawNode{ ID: "hostC", @@ -108,9 +104,8 @@ func TestToDirectoryNode_IgnoresSRVPort(t *testing.T) { } } -// TestToDirectoryNodeSkipsWithoutUUID: a record with no uuid= can't be keyed by -// a stable identity, so it's skipped (ok=false) rather than keyed by the -// hostname — node identity is the UUID everywhere, no name fallback. +// Candidate authentication needs a host UUID claim, even though the claim +// does not determine the independent discovery record ID or grant trust. func TestToDirectoryNode_SkipsWithoutUUID(t *testing.T) { raw := RawNode{ ID: "hostB", @@ -121,14 +116,8 @@ func TestToDirectoryNode_SkipsWithoutUUID(t *testing.T) { assert.False(t, ok) } -// TestToDirectoryNodeIPFallback: with a uuid= but no ip=, every advertised address -// becomes a candidate and the canonical one is the ranker's head. The private -// blocks tie, so this is an arbitrary but stable choice — deliberately so. Which -// private block a node's real network uses is not knowable from the address, and -// the previous preference for 192.168 over 10.x is what made a multi-homed host -// publish a two-host direct-connect link instead of its LAN. Both addresses are -// kept, and a consumer that must connect resolves it by connecting. -func TestToDirectoryNode_UsesRecordIPWhenAddressesAreAbsent(t *testing.T) { +// Without TXT addresses, retain every resolved candidate in its published order. +func TestToDirectoryNode_UsesResolvedAddressesWithoutIPClaims(t *testing.T) { raw := RawNode{ ID: "hostB", Addresses: []string{"10.221.0.9", "192.168.1.20"}, @@ -141,10 +130,7 @@ func TestToDirectoryNode_UsesRecordIPWhenAddressesAreAbsent(t *testing.T) { assert.False(t, n.Clustered()) } -// TestToDirectoryNodePreservesPublishedOrder: a node's own ips= order survives -// projection, and its ip= stays canonical even though the advertised address list -// leads with something else. This is the multi-homed fix at the collapse point — -// the node ranked these with evidence no observer has. +// The TXT address order supplies candidate preference over resolved address order. func TestToDirectoryNode_PreservesPublishedOrder(t *testing.T) { raw := RawNode{ ID: "spark", @@ -160,8 +146,7 @@ func TestToDirectoryNode_PreservesPublishedOrder(t *testing.T) { assert.Equal(t, []string{"10.172.54.70", "192.168.240.2"}, n.IPs) } -// TestToDirectoryNodeUnionsUnpublishedAddresses: an address the browse resolved -// but the node did not rank is kept as a fallback, appended rather than promoted. +// Resolved addresses absent from TXT remain available as fallback candidates. func TestToDirectoryNode_UnionsUnpublishedAddresses(t *testing.T) { raw := RawNode{ ID: "hostC", diff --git a/services/nvpair-node-scanner/enrich_hosts.go b/services/nvpair-node-scanner/enrich_hosts.go index bb218eae9..3ebeaca84 100644 --- a/services/nvpair-node-scanner/enrich_hosts.go +++ b/services/nvpair-node-scanner/enrich_hosts.go @@ -21,9 +21,9 @@ type modelInventory struct { certificate *x509.Certificate } -// hostKey identifies one node's endpoint for one service. Node-info and the engine -// manager are different ports and can be reachable at different addresses, so what -// answered for one says nothing about the other. +// hostKey identifies address memory for one record and service. Node-info uses +// discovery record IDs to keep candidates separate from verified peers; model +// reads use the canonical host UUID. Each service remembers its own working IP. type hostKey struct { hostUUID string service noderec.ServiceKey diff --git a/services/nvpair-node-scanner/enrich_hosts_test.go b/services/nvpair-node-scanner/enrich_hosts_test.go index 5cf849c52..516d85530 100644 --- a/services/nvpair-node-scanner/enrich_hosts_test.go +++ b/services/nvpair-node-scanner/enrich_hosts_test.go @@ -163,24 +163,47 @@ func TestAskTogether_PrefersRankOverArrivalOrder(t *testing.T) { require.Equal(t, "10.0.0.5", host, "askTogether (%v, %v)", host, ok) } -// TestAskTogetherPaysOneTimeoutForTheWholeList: an address that neither answers nor -// refuses costs a whole fetch timeout, and asking in sequence paid that per address -// — up to modelsFetchTimeout each — before reaching one that works. -func TestAskTogether_PaysOneTimeoutForTheWholeList(t *testing.T) { - const stall = 150 * time.Millisecond - net := newAnswering("10.0.0.9") - for _, host := range []string{"192.168.240.1", "169.254.7.7", "172.17.0.1"} { - net.slow[host] = stall - } +// Every address must be asked before any response is released. Sequential +// probing would wait forever on the first address instead of reaching the rest. +func TestAskTogether_ProbesAddressesConcurrently(t *testing.T) { hosts := []string{"192.168.240.1", "169.254.7.7", "172.17.0.1", "10.0.0.9"} - - start := time.Now() - host, _, ok := askTogether(hosts, net.ask) - elapsed := time.Since(start) - - require.True(t, ok, "askTogether (%v, %v)", host, ok) - require.Equal(t, "10.0.0.9", host, "askTogether (%v, %v)", host, ok) - assert.LessOrEqual(t, elapsed, 2*stall, "asking four addresses took (%v, %v)", elapsed, stall) + entered := make(chan string, len(hosts)) + release := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + type probeResult struct { + host string + ok bool + } + finished := make(chan probeResult, 1) + go func() { + host, _, ok := askTogether(hosts, func(host string) (string, bool) { + entered <- host + <-release + return "inventory", host == "10.0.0.9" + }) + finished <- probeResult{host: host, ok: ok} + }() + + var asked []string + for range hosts { + select { + case host := <-entered: + asked = append(asked, host) + case <-time.After(2 * time.Second): + require.FailNow(t, "addresses were not probed concurrently") + } + } + assert.ElementsMatch(t, hosts, asked) + releaseOnce.Do(func() { close(release) }) + + select { + case result := <-finished: + assert.True(t, result.ok) + assert.Equal(t, "10.0.0.9", result.host) + case <-time.After(2 * time.Second): + require.FailNow(t, "address probing did not finish after responses were released") + } } // TestAskTogetherSkipsBlankHosts guards the published-list edge: a record can carry diff --git a/services/nvpair-node-scanner/node_info_outage_test.go b/services/nvpair-node-scanner/node_info_outage_test.go index 099e33d2a..75fff9832 100644 --- a/services/nvpair-node-scanner/node_info_outage_test.go +++ b/services/nvpair-node-scanner/node_info_outage_test.go @@ -28,10 +28,7 @@ func captureLogs(t *testing.T) *bytes.Buffer { return buf } -// TestNodeInfoOutageIsStatedOnceAtEachEnd pins the reporting to the transitions. -// The sweep repeats for as long as the node is advertised, so reporting every -// attempt is what made this unreadable — and reporting none of them is what hid -// it entirely. +// Repeated polling reports outage and recovery transitions once each. func TestNodeInfoOutage_IsStatedOnceAtEachEnd(t *testing.T) { logs := captureLogs(t) d := &daemon{} @@ -49,8 +46,7 @@ func TestNodeInfoOutage_IsStatedOnceAtEachEnd(t *testing.T) { require.Equal(t, 1, strings.Count(logs.String(), "node-info is answering again"), "recovery reported") } -// TestNodeInfoSilenceWhenNothingChanges keeps the steady state quiet: a peer that -// has always answered has nothing to say about itself. +// A peer that remains available must not produce repetitive log messages. func TestNodeInfoOutage_EmitsNothingWhileAvailabilityIsUnchanged(t *testing.T) { logs := captureLogs(t) d := &daemon{} diff --git a/services/nvpair-node-scanner/refresh_test.go b/services/nvpair-node-scanner/refresh_test.go index 4e4c9159d..621739aad 100644 --- a/services/nvpair-node-scanner/refresh_test.go +++ b/services/nvpair-node-scanner/refresh_test.go @@ -83,12 +83,9 @@ func portFromURL(t *testing.T, raw string) int { return p } -// newRefreshTestDaemon builds a daemon wired for the refresh path only: a real -// directory + modelsHTTP client and initialized caches, no codec/responder (emit -// is nil-safe and the refresh path never advertises). The registry is stamped -// with a self uuid that no peer test seeds, so refreshNodeModels dials each peer -// at its advertised address (the self-loopback branch is exercised separately in -// TestRefreshSelfDialsLoopback). +// newRefreshTestDaemon initializes model reconciliation without announcements or +// IPC. Its loopback fixtures exercise inventory changes; authenticated remote +// behavior uses newDiscoveryPeerFixture. func newRefreshTestDaemon() *daemon { return &daemon{ reg: newRegistry("self-node-uuid", "", selfAddrs("127.0.0.1")), @@ -114,10 +111,7 @@ func seedEMNode(d *daemon, hostUUID, ip string, emPort int) { }) } -// TestRefreshPopulatesInitialEmptyInventory covers the primary bug: a known em -// peer whose first enrichment returned nothing converges once its engine serves -// models, with no mDNS record change — a single refresh fetch populates the -// directory and reports a change. +// Model inventory converges without requiring a changed discovery announcement. func TestRefresh_PopulatesInitialEmptyInventory(t *testing.T) { d := newRefreshTestDaemon() _, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -153,14 +147,8 @@ func TestRefreshModels_FallsBackToASecondPublishedAddress(t *testing.T) { require.Equal(t, []string{"reachable-model"}, got.Models, "Models") } -// TestRefreshSelfDialsLoopback is the regression for the periodic model -// refresh: our OWN node must be dialed over loopback, not its advertised LAN -// ip=, so a pull/delete/late-engine-start keeps converging after startup even -// when inbound to the LAN address is firewalled (the Windows failure this -// addresses). The stub listens ONLY on loopback while the self entry advertises -// an unreachable LAN address, so the fetch can only succeed via the loopback -// dial. The stored (advertised) address is still used as applyModels' stale- -// address guard, so it must be preserved. +// The owned engine-manager listens on loopback while the directory advertises +// an unreachable LAN address. A local refresh must preserve that advertised IP. func TestRefreshSelf_DialsLoopback(t *testing.T) { d := newRefreshTestDaemon() // reg self uuid = "self-node-uuid" _, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -305,9 +293,7 @@ func TestRefreshModels_ReportsDownloadsAndDeletionsButNotUnchangedInventory(t *t assert.Equal(t, map[string][]string{"ollama": {}}, got.ModelsByEngine, "ModelsByEngine after last delete") } -// TestRefreshReorderOnlyIsNotAChange asserts the compare is order-insensitive: a -// response that returns the same models in a different order is not treated as a -// change (no spurious update). +// Reordering the same installed models must not emit another directory update. func TestRefreshReorderOnly_IsNotAChange(t *testing.T) { d := newRefreshTestDaemon() stub, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -323,9 +309,7 @@ func TestRefreshReorderOnly_IsNotAChange(t *testing.T) { assert.False(t, d.refreshNodeModels("peer-R", "127.0.0.1", port, ""), "a mere reordering of the same models should not report a change") } -// TestRefreshFetchFailureRetainsInventory covers requirement 5: a transient -// fetch failure preserves the last successful flat and per-engine inventories -// (no blanking) and reports no change. +// A failed refresh preserves both flat and per-engine inventories. func TestRefreshFetchFailure_RetainsInventory(t *testing.T) { d := newRefreshTestDaemon() stub, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -343,9 +327,7 @@ func TestRefreshFetchFailure_RetainsInventory(t *testing.T) { assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, got.ModelsByEngine, "ModelsByEngine after failure") } -// TestRefreshPartialModelsByEngine covers requirement 4: a partial -// modelsByEngine response (attribution covering fewer models than the flat -// union, as a mixed-version peer might send) is propagated exactly as returned. +// A peer may attribute fewer models to engines than it includes in the union. func TestRefreshModels_PreservesInventoryForUnreportedEngines(t *testing.T) { d := newRefreshTestDaemon() _, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -359,8 +341,7 @@ func TestRefreshModels_PreservesInventoryForUnreportedEngines(t *testing.T) { assert.Equal(t, map[string][]string{"ollama": {"x"}}, got.ModelsByEngine, "ModelsByEngine") } -// TestRefreshPopulatesLoadedByEngine covers Phase 2: a peer's loadedByEngine is -// carried onto the directory node from the same /v1/models fetch. +// The same model fetch supplies the peer's installed and loaded inventories. func TestRefresh_PopulatesLoadedByEngine(t *testing.T) { d := newRefreshTestDaemon() _, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -375,10 +356,7 @@ func TestRefresh_PopulatesLoadedByEngine(t *testing.T) { assert.Equal(t, map[string][]string{"ollama": {"llama3:8b"}, "lmstudio": {}}, got.LoadedByEngine, "LoadedByEngine") } -// TestRefreshLoadedOnlyChangeReportsChange covers the loaded-set reconcile: when -// only loadedByEngine changes (a JIT load / TTL eviction with the same installed -// list), the refresh reports a change and updates the field so remote cards stay -// live. +// Loading a model changes the directory even when installed models stay the same. func TestRefreshLoadedOnlyChange_ReportsChange(t *testing.T) { d := newRefreshTestDaemon() stub, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -408,8 +386,7 @@ func TestRefreshLoadedOnlyChange_ReportsChange(t *testing.T) { assert.False(t, d.refreshNodeModels("peer-M", "127.0.0.1", port, ""), "an identical loaded set should not report a change") } -// TestRefreshLoadedRetainedOnFailure confirms a transient fetch failure preserves -// the last successful loadedByEngine rather than blanking it. +// A failed refresh must not clear the last accepted loaded-model inventory. func TestRefreshModels_RetainsLoadedModelsAfterFetchFailure(t *testing.T) { d := newRefreshTestDaemon() stub, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -426,43 +403,7 @@ func TestRefreshModels_RetainsLoadedModelsAfterFetchFailure(t *testing.T) { assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, got.LoadedByEngine, "LoadedByEngine after failure") } -// TestApplyModelsGuard covers requirement 6 at the directory layer: a completed -// fetch is discarded (ok == false) when the node was removed or re-addressed -// while it was in flight, so it can neither resurrect a removed node nor -// overwrite a re-addressed one. -func TestApplyModels_RejectsRemovedNodeAndStaleAddress(t *testing.T) { - d := newDirectory() - - // Removed mid-sweep: node absent -> not resurrected. - _, changed, ok := d.applyModels("ghost", "127.0.0.1", 14322, []string{"a"}, nil, nil) - assert.False(t, ok, "applyModels on an absent node = (changed (%v, %v)", changed, ok) - assert.False(t, changed, "applyModels on an absent node = (changed (%v, %v)", changed, ok) - _, present := d.get("ghost") - assert.False(t, present, "applyModels must not resurrect a removed node") - - // Re-addressed mid-sweep: node present but IP changed -> not overwritten. - d.upsert(noderec.DirectoryNode{ - HostUUID: "peer-E", - IP: "10.0.0.9", - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14322}}, - Models: []string{"old"}, - }) - _, changed, ok = d.applyModels("peer-E", "127.0.0.1", 14322, []string{"new"}, nil, nil) - assert.False(t, ok, "applyModels with a stale IP = (changed (%v, %v)", changed, ok) - assert.False(t, changed, "applyModels with a stale IP = (changed (%v, %v)", changed, ok) - // em port changed -> also discarded. - _, changed, ok = d.applyModels("peer-E", "10.0.0.9", 99999, []string{"new"}, nil, nil) - assert.False(t, ok, "applyModels with a stale em port = (changed (%v, %v)", changed, ok) - assert.False(t, changed, "applyModels with a stale em port = (changed (%v, %v)", changed, ok) - got, _ := d.get("peer-E") - assert.Equal(t, []string{"old"}, got.Models, "Models after guarded rejects") -} - -// TestRefreshNodeModelsRemovedMidSweep exercises the guard through the daemon's -// per-node path: a node evicted between the sweep snapshot and the fetch's -// completion is not resurrected in the directory (the requirement-6 guarantee). -// The cache may hold the fetched value (it is written before the directory -// guard, as last-known-good), so this asserts only the directory invariant. +// A missing node must stay absent when a periodic sweep reaches its old target. func TestRefreshNodeModels_DoesNotRestoreNodeRemovedDuringSweep(t *testing.T) { d := newRefreshTestDaemon() _, port := startModelsStub(t, http.StatusOK, map[string]any{ @@ -476,13 +417,8 @@ func TestRefreshNodeModels_DoesNotRestoreNodeRemovedDuringSweep(t *testing.T) { assert.False(t, present, "an in-flight result must not resurrect a removed node") } -// TestRefreshOnceConcurrentSlowPeers covers requirement 7: multiple slow peers -// are refreshed concurrently, so one slow peer does not consume another's -// timeout budget. It proves concurrency deterministically with an entry barrier -// rather than a wall-clock threshold: both stub handlers must be ENTERED before -// either is released. A serial sweep would block inside the first handler and -// never enter the second, so the barrier wait fails fast. The time.After bounds -// are failure ceilings, not success-timing assertions, so the test isn't flaky. +// Both peer handlers must be entered before either is released. This proves +// concurrency without requiring successful requests to meet a timing threshold. func TestRefreshModels_ConcurrentAuthenticatedPeers(t *testing.T) { first := newDiscoveryPeerFixture(t, "peer-1") first.discover(t) @@ -529,3 +465,37 @@ func TestRefreshModels_ConcurrentAuthenticatedPeers(t *testing.T) { type modelInventoryResponse struct { Models []string `json:"models"` } + +func TestApplyModels_DoesNotRestoreRemovedNode(t *testing.T) { + d := newDirectory() + + _, changed, ok := d.applyModels("ghost", "127.0.0.1", 14322, []string{"a"}, nil, nil) + assert.False(t, ok) + assert.False(t, changed) + _, present := d.get("ghost") + assert.False(t, present) +} + +func TestApplyModels_RejectsChangedEndpoint(t *testing.T) { + test := func(name, ip string, port int) { + t.Run(name, func(t *testing.T) { + d := newDirectory() + d.upsert(noderec.DirectoryNode{ + HostUUID: "peer-E", + IP: "10.0.0.9", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14322}}, + Models: []string{"old"}, + }) + + _, changed, ok := d.applyModels("peer-E", ip, port, []string{"new"}, nil, nil) + + assert.False(t, ok) + assert.False(t, changed) + node, present := d.get("peer-E") + require.True(t, present) + assert.Equal(t, []string{"old"}, node.Models) + }) + } + test("address changes", "127.0.0.1", 14322) + test("port changes", "10.0.0.9", 14323) +} diff --git a/services/nvpair-node-scanner/scanner.go b/services/nvpair-node-scanner/scanner.go index 576f044b0..ed1491b23 100644 --- a/services/nvpair-node-scanner/scanner.go +++ b/services/nvpair-node-scanner/scanner.go @@ -37,11 +37,8 @@ type ReadyParams struct { Version string `json:"version"` } -// Scanner is the stdio/JSON-RPC front end for the promoted discovery daemon. -// Post-cutover there is no legacy _nvpair-node-info browse or node/* stream: the -// daemon advertises one _nvpair-node record, browses _nvpair-node, maintains the -// directory, and serves the discovery:* RPC. This wrapper owns the process -// lifecycle and routes stdio frames into the daemon. +// Scanner owns the discovery daemon's process lifecycle and routes stdio +// JSON-RPC frames into its discovery:* handlers. type Scanner struct { codec *Codec daemon *daemon diff --git a/services/nvpair-node-scanner/telemetry_test.go b/services/nvpair-node-scanner/telemetry_test.go index 2b432ba5b..87d924e3d 100644 --- a/services/nvpair-node-scanner/telemetry_test.go +++ b/services/nvpair-node-scanner/telemetry_test.go @@ -179,17 +179,17 @@ func TestTelemetryIntervalForNode_IsStableAndBounded(t *testing.T) { } func TestTelemetryRetryDelay_BoundsBackoff(t *testing.T) { - test := func(name string, failures int, want time.Duration) { + test := func(name string, failures int, expected time.Duration) { t.Run(name, func(t *testing.T) { - assert.Equal(t, want, telemetryRetryDelay(failures)) + assert.Equal(t, expected, telemetryRetryDelay(failures)) }) } - test("initial delay", 0, 2*time.Second) - test("one failure", 1, 4*time.Second) - test("two failures", 2, 8*time.Second) - test("three failures", 3, 16*time.Second) - test("maximum delay", 4, 30*time.Second) - test("delay stays capped", 100, 30*time.Second) + test("no failures", 0, 2*time.Second) + test("first failure", 1, 4*time.Second) + test("second failure", 2, 8*time.Second) + test("third failure", 3, 16*time.Second) + test("fourth failure reaches maximum", 4, 30*time.Second) + test("later failures retain maximum", 100, 30*time.Second) } func TestTelemetryRetryGate_BacksOffAndResets(t *testing.T) { @@ -231,8 +231,8 @@ func TestTelemetryRetryGateChangedTarget_WaitsForActiveAttempt(t *testing.T) { require.True(t, ok, "old endpoint failure backed off the changed endpoint") }) } - test("hosts", telemetryTargetKey([]string{"10.0.0.1"}, 14318)) - test("port", telemetryTargetKey([]string{"192.0.2.1"}, 14319)) + test("address changes", telemetryTargetKey([]string{"10.0.0.1"}, 14318)) + test("port changes", telemetryTargetKey([]string{"192.0.2.1"}, 14319)) } func TestTelemetryRetryGateRemoveRediscover_PreservesActiveClaim(t *testing.T) { diff --git a/services/nvpair-proxy/proxy_test.go b/services/nvpair-proxy/proxy_test.go index 916895970..205312903 100644 --- a/services/nvpair-proxy/proxy_test.go +++ b/services/nvpair-proxy/proxy_test.go @@ -80,47 +80,18 @@ func TestSubscribedToNode_AcceptsLocalRecordsWithoutPeerTrust(t *testing.T) { assert.Equal(t, enginePort, projected.Port) } -// TestNodeCandidates covers the deterministic, loopback-first ordering and -// de-duplication of a node's advertised addresses. Local-address rewriting is -// not exercised (it depends on the host's interfaces); these addresses are -// chosen to be non-local. -func TestNodeCandidates_UsesPublishedAddressOrder(t *testing.T) { - tests := []struct { - name string - node Node - want []string - }{ - { - name: "sorted deterministically", - node: Node{Addresses: []string{"192.0.2.20", "192.0.2.10"}, Port: 11434}, - want: []string{"192.0.2.10:11434", "192.0.2.20:11434"}, - }, - { - name: "loopback floated to front", - node: Node{Addresses: []string{"192.0.2.10", "127.0.0.1"}, Port: 11434}, - want: []string{"127.0.0.1:11434", "192.0.2.10:11434"}, - }, - { - name: "deduplicated", - node: Node{Addresses: []string{"192.0.2.10", "192.0.2.10"}, Port: 11434}, - want: []string{"192.0.2.10:11434"}, - }, - { - name: "fallback to host", - node: Node{Host: "gpu-host.lan", Port: 11434}, - want: []string{"gpu-host.lan:11434"}, - }, - { - name: "ipv6 bracketed", - node: Node{Addresses: []string{"2001:db8::1"}, Port: 11434}, - want: []string{"[2001:db8::1]:11434"}, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, nodeCandidates(tc.node), "nodeCandidates") +// Documentation addresses avoid host-dependent local-interface rewriting. +func TestNodeCandidates_RanksAndDeduplicatesAddresses(t *testing.T) { + test := func(name string, node Node, want []string) { + t.Run(name, func(t *testing.T) { + assert.Equal(t, want, nodeCandidates(node)) }) } + test("sorted deterministically", Node{Addresses: []string{"192.0.2.20", "192.0.2.10"}, Port: 11434}, []string{"192.0.2.10:11434", "192.0.2.20:11434"}) + test("loopback first", Node{Addresses: []string{"192.0.2.10", "127.0.0.1"}, Port: 11434}, []string{"127.0.0.1:11434", "192.0.2.10:11434"}) + test("deduplicated", Node{Addresses: []string{"192.0.2.10", "192.0.2.10"}, Port: 11434}, []string{"192.0.2.10:11434"}) + test("fallback to host", Node{Host: "gpu-host.lan", Port: 11434}, []string{"gpu-host.lan:11434"}) + test("ipv6 bracketed", Node{Addresses: []string{"2001:db8::1"}, Port: 11434}, []string{"[2001:db8::1]:11434"}) } func TestUUIDFromTXT_ExtractsNodeIdentity(t *testing.T) { @@ -128,53 +99,19 @@ func TestUUIDFromTXT_ExtractsNodeIdentity(t *testing.T) { assert.Equal(t, "", uuidFromTXT([]string{"models=a;b"})) } -// TestNodeURL covers the URL-construction part of nodeURL — specifically -// that IPv6 literals are bracket-wrapped and IPv4/hostnames remain -// byte-identical to the pre-JoinHostPort implementation. The -// local-address shortcut is not exercised here because it depends on -// the host's network interfaces (see init() in proxy.go) and would be -// flaky across environments. +// Documentation addresses avoid host-dependent local-interface rewriting while +// checking the host, port, and IPv6 brackets used to construct each URL. func TestNodeURL_FormatsIPv4IPv6AndServicePort(t *testing.T) { - // Pick addresses that are unlikely to appear on any local interface. - tests := []struct { - name string - node Node - wantHost string - wantURL string - }{ - { - name: "ipv4", - node: Node{Addresses: []string{"192.0.2.10"}, Port: 11434}, - wantHost: "192.0.2.10:11434", - wantURL: "http://192.0.2.10:11434", - }, - { - name: "ipv6", - node: Node{Addresses: []string{"2001:db8::1"}, Port: 11434}, - wantHost: "[2001:db8::1]:11434", - wantURL: "http://[2001:db8::1]:11434", - }, - { - name: "hostname", - node: Node{Addresses: []string{"gpu-host.lan"}, Port: 11434}, - wantHost: "gpu-host.lan:11434", - wantURL: "http://gpu-host.lan:11434", - }, - { - // Empty Addresses slice — nodeURL should fall back to Host. - name: "fallback to Host", - node: Node{Host: "gpu-host.lan", Port: 11434}, - wantHost: "gpu-host.lan:11434", - wantURL: "http://gpu-host.lan:11434", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - u := nodeURL(tc.node) - require.NotNil(t, u, "nodeURL returned nil") - assert.Equal(t, tc.wantHost, u.Host, "Host") - assert.Equal(t, tc.wantURL, u.String()) + test := func(name string, node Node, wantHost, wantURL string) { + t.Run(name, func(t *testing.T) { + u := nodeURL(node) + require.NotNil(t, u) + assert.Equal(t, wantHost, u.Host) + assert.Equal(t, wantURL, u.String()) }) } + test("ipv4", Node{Addresses: []string{"192.0.2.10"}, Port: 11434}, "192.0.2.10:11434", "http://192.0.2.10:11434") + test("ipv6", Node{Addresses: []string{"2001:db8::1"}, Port: 11434}, "[2001:db8::1]:11434", "http://[2001:db8::1]:11434") + test("hostname", Node{Addresses: []string{"gpu-host.lan"}, Port: 11434}, "gpu-host.lan:11434", "http://gpu-host.lan:11434") + test("fallback to host", Node{Host: "gpu-host.lan", Port: 11434}, "gpu-host.lan:11434", "http://gpu-host.lan:11434") } diff --git a/services/nvpair-proxy/subscribed_test.go b/services/nvpair-proxy/subscribed_test.go index c16e642ca..d7b082d21 100644 --- a/services/nvpair-proxy/subscribed_test.go +++ b/services/nvpair-proxy/subscribed_test.go @@ -11,11 +11,8 @@ import ( "nvpair-shared/noderec" ) -// TestSubscribedOverlayMerge covers the relay-sourced routing overlay: the broker -// pushes the full filtered set, SetSubscribed replaces the overlay wholesale, and -// subscribed nodes merge into Nodes() alongside the separate manual overlay. A -// later snapshot omitting a node drops it while manual nodes survive. The browser -// isn't running, so Nodes() reflects only the overlays. +// Replacing the relay snapshot removes departed peers while preserving the +// independently configured manual overlay. func TestSubscribedOverlay_MergesDiscoveryAndManualNodes(t *testing.T) { d := NewDiscovery() @@ -40,9 +37,8 @@ func TestSubscribedOverlay_MergesDiscoveryAndManualNodes(t *testing.T) { require.True(t, d.IsManual("m1"), "manual node m1 should survive a subscribed-set replace") } -// TestSubscribedDiff covers the diff SetSubscribed returns so the proxy can emit -// node/discovered|updated|removed for the relay-fed set (the signal the UI uses to -// show which peers run this engine). +// The returned changes drive node/discovered, node/updated, and node/removed +// notifications so clients learn which peers currently run this engine. func TestSubscribedDiff_ReportsAddedChangedAndRemovedNodes(t *testing.T) { d := NewDiscovery() @@ -85,14 +81,8 @@ func TestSubscribedDiff_ReportsAddedChangedAndRemovedNodes(t *testing.T) { require.Equal(t, "s1", rem[0].ID, "omitted node: want removed=[s1], got disc (%v, %v, %v)", disc, upd, rem) } -// TestSubscribedToNode covers the DirectoryNode -> routable Node projection: -// this engine's port is read from its own service key, and a node lacking that -// key, or an IP, is rejected. -// -// The two negative fixtures matter for different reasons and the old suites -// carried one each — Ollama rejected a non-engine service, LM Studio rejected -// the other engine. Both run for both engines here, because the second is what -// regresses if the projection ever loses its service-key filter. +// The proxy projects only its engine's service endpoint and model inventory. +// Other engine services and non-engine services cannot supply that endpoint. func TestSubscribedToNode_ProjectsDiscoveryFields(t *testing.T) { forEachEngine(t, func(t *testing.T, tc engineCase) { const hostUUID = "uuid-a" @@ -178,10 +168,8 @@ func TestSubscribedToNode_ProjectsDiscoveryFields(t *testing.T) { }) } -// TestSubscribedToNodeKeysByHostUUID: the routable Node keys on the stable -// hostUuid, not the hostname — so routing, scheduledOn, node selection, and the -// scheduler's priority list all survive a PC rename and never conflate two -// same-named machines. Host stays the hostname for display. +// The stable host UUID keeps routing and scheduling independent of PC renames +// and same-named machines. Host preserves the hostname for display. func TestSubscribedToNode_KeysByHostUUID(t *testing.T) { forEachEngine(t, func(t *testing.T, tc engineCase) { const uuid = "11111111-1111-1111-1111-111111111111" diff --git a/services/nvpair-tui/README.md b/services/nvpair-tui/README.md index c937c020f..9feb0266b 100644 --- a/services/nvpair-tui/README.md +++ b/services/nvpair-tui/README.md @@ -117,8 +117,9 @@ enriches from each peer's engine manager — no extra request. This machine's co from `engine:models` and stay live through `engine:models-changed`. The detail screen also polls the node's own `/v1/node-info` endpoint over HTTP -for GPU, CPU, and memory. That is the one reading the broker's JSON-RPC surface -does not carry, and only the open node is polled. +for GPU, CPU, and memory. Only the open node is polled, and public readings are +used for display. Verified manual TLS configurations use the manual worker's +authenticated hardware reading instead. ## Keys diff --git a/services/nvpair-tui/ui/nodesmodel.go b/services/nvpair-tui/ui/nodesmodel.go index 54f5b41f2..ab32f0867 100644 --- a/services/nvpair-tui/ui/nodesmodel.go +++ b/services/nvpair-tui/ui/nodesmodel.go @@ -99,8 +99,8 @@ func (m nodeMembership) relationship() string { // - cluster — membership state for peers we have paired with // - manual — user-added entries discovery cannot see, plus their probes // -// One machine can appear in all three. Merging on a stable key is what stops it -// rendering as three separate rows. +// Verified host identities join matching feeds. Unverified discoveries retain +// separate record keys, and unverified manual configurations retain manual keys. type nodeRow struct { identityVerification noderec.IdentityVerification key string @@ -146,8 +146,8 @@ type nodeFeeds struct { // Discovery is the base layer because it carries the richest record. The // cluster roster then contributes membership and, critically, keeps a member // listed even when discovery has dropped it — that node is offline, not gone. -// Manual entries are matched by address so a hand-added host that discovery -// later finds does not appear twice. +// Manual entries join matching authenticated host identities. Names and shared +// addresses never join unverified discoveries to those records. func mergeNodes(in nodeFeeds) []nodeRow { byKey := make(map[string]*nodeRow, len(in.discovered)+len(in.members)+len(in.manual)) order := make([]string, 0, len(byKey)) diff --git a/services/nvpair-tui/ui/nodeswire.go b/services/nvpair-tui/ui/nodeswire.go index ecd4ab853..805043d95 100644 --- a/services/nvpair-tui/ui/nodeswire.go +++ b/services/nvpair-tui/ui/nodeswire.go @@ -88,19 +88,19 @@ type manualNode struct { ID string `json:"id"` Name string `json:"name"` Address string `json:"address"` - // HostUUID is the machine's own identity, read from its node-info. It is - // what joins a hand-typed hostname to the same machine discovered by IP. + // HostUUID comes from authenticated node-info and can join a verified + // manual entry to the same host discovered by IP. HostUUID string `json:"hostUuid"` OllamaUp bool `json:"ollama_up"` LMStudioUp bool `json:"lmstudio_up"` NodeInfoUp bool `json:"node_info_up"` - // NodeInfoPort is the port the probe reached node-info on, and TLSEnabled - // whether that was over TLS. + // NodeInfoPort is the attempted node-info port. TLSEnabled echoes the + // configured TLS port hint; service hardware probes always use mTLS. NodeInfoPort int `json:"node_info_port"` TLSEnabled bool `json:"tls_enabled"` - // The worker's own hardware reading from that probe. For a TLS node it is - // the only one to be had: the worker holds the backend's trust to reach - // it, and this client holds none. + // Authenticated hardware readings are available from the manual worker. + // The TUI holds no cluster certificate and uses these for manual TLS + // configurations rather than polling their TLS endpoints itself. GPUs []noderec.GPUInfo `json:"gpus"` CPU *noderec.CPUInfo `json:"cpu"` Memory *noderec.MemoryInfo `json:"memory"` diff --git a/services/nvpair-tui/ui/nodetelemetry.go b/services/nvpair-tui/ui/nodetelemetry.go index 5af3e2fd1..d55e56789 100644 --- a/services/nvpair-tui/ui/nodetelemetry.go +++ b/services/nvpair-tui/ui/nodetelemetry.go @@ -19,11 +19,9 @@ import ( tea "github.com/charmbracelet/bubbletea" ) -// The node-info HTTP contract. Hardware telemetry is the one thing the broker's -// JSON-RPC surface does not carry: it keeps GPU, CPU, and memory readings -// internally but does not put them on the discovery wire, so a client that wants -// them has to ask each node directly. The desktop app does the same thing, and -// these values match its cadence so the two behave alike. +// Public node-info HTTP reads supply hardware displays independently of the +// service measurements used by the broker and scheduler. These frontend reads +// continue for paired and unpaired nodes, with the desktop's polling cadence. const ( nodeInfoPath = "/v1/node-info" nodeInfoDefaultPort = 14318 diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index b9a5c2848..f422dadce 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -327,8 +327,8 @@ type Broker struct { engineRelaySubID int // manualMu guards manualNodeKeys, mapping each manual node's id to the - // discovery-store key it currently occupies — its real hostUuid once - // node-info reports one, else its manual id until then. Tracked + // discovery-store key it currently occupies — its authenticated hostUuid, + // or its independent manual configuration key. Tracked // per manual id so the broker can (a) re-key the store entry when a manual // node's node-info first reveals its stable UUID, and (b) evict the right // key on removal or a manual-nodes crash — since the store key is no longer @@ -337,8 +337,7 @@ type Broker struct { // manual claim is dropped only once the last alias for a key is gone, and // when an alias that a key was currently projecting from leaves, a surviving // alias is reprojected from manualNodeStatuses (its last-seen payload) so the - // removed alias's address/models don't linger (the survivor's own periodic - // probe emits no update while its status is unchanged). + // removed alias's address/models don't linger until the next accepted probe. manualMu sync.Mutex manualNodeKeys map[string]string manualNodeStatuses map[string]manualNodeStatusEntry @@ -1500,11 +1499,8 @@ func (b *Broker) applyClusterIdentityChange() { } } -// forwardEngineNotification / forwardManualNodesNotification / -// forwardSettingsNotification are the per-worker notification hooks. For -// now they only demux errors:report / errors:clear into the nvpair-errors -// pipeline; the rest of each worker's notification stream is logged and -// dropped until its control-plane relay is wired in. +// forwardEngineNotification handles engine-manager's settings and discovery +// relay requests, routes service errors, and forwards client-facing events. func (b *Broker) forwardEngineNotification(method string, params json.RawMessage) { if method == "engine:settings-request" { b.handleSettingsRelay(params) @@ -1654,14 +1650,9 @@ func (b *Broker) handleManualNodesNotification(method string, params json.RawMes } } -// trackManualNode records the discovery-store key a manual node currently -// occupies and reports the previous key when it changed (a manual node's -// node-info first reported its real hostUuid, moving it off the manual id). On a -// rekey it drops the manual claim under the old key (the scanner's claim, if -// any, survives); the caller rebridges the proxy overlay off the old key. // upsertManualNode ingests a manual node status: it records the alias's payload, // keys the discovery store by the node's operational identity (its hostUuid once -// node-info reports one, else the manual id), and bridges the proxy candidate +// authenticated over pinned mTLS, else the manual id), and bridges the proxy candidate // under that same key so scheduler priority and scheduledOn resolve to it. If // the alias's key changed (node-info revealed its real UUID) the // old key is reprojected from a surviving alias or released. @@ -1706,9 +1697,8 @@ func (b *Broker) upsertManualNodeAt(s manualNodeStatus, receivedAt time.Time) { // removeManualNode forgets a manual alias, then reprojects or releases the key // it occupied. When another alias for the same machine still owns the key, the // node stays and is reprojected from that survivor's last-seen payload — so the -// removed alias's address/models don't linger (the survivor's steady-state probe -// emits no update to restore itself); only the final alias's removal drops the -// store/proxy claim. +// removed alias's address/models don't linger until the next accepted probe. +// Only the final alias's removal drops the store/proxy claim. func (b *Broker) removeManualNode(id string) { b.manualMu.Lock() key, ok := b.manualNodeKeys[id] @@ -2855,10 +2845,8 @@ func (b *Broker) forwardClusterManagerNotification(method string, params json.Ra } b.applyClusterIdentityChange() } - // A pairing or removal changed the pin set. The cluster-scoped workers read - // their gates live and need nothing, but the node-scanner caches a per-peer - // trusted annotation it can only otherwise refresh when that peer's mDNS - // record moves — which a peer already advertising when we joined never does. + // A pairing or removal changed the pin set. Withdraw stale proofs and + // measurements, then ask scanner to authenticate against current pins. if method == "cluster:trust-changed" { b.applyClusterTrustChange() } diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 932066153..8dd8b998c 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -134,8 +134,8 @@ const ( sourceManual ) -// storedNode is the broker's in-memory record per discovered node, keyed by -// hostUuid. It keeps each contributing source's last full EnrichedNode payload +// storedNode keeps a verified host by UUID or an unverified discovery by its +// separate record ID. It keeps each source's last full EnrichedNode payload // (including GPU / CPU / memory the wire doesn't expose) so a record can be // re-projected from the surviving source when the other is removed. lastSeen is // bumped on every discovered/updated event, giving clients a freshness signal. @@ -250,8 +250,8 @@ func (sn storedNode) hasSource(source nodeSource) bool { // discoveryStore is the broker's live view of network-discovered nodes. // It's the single source of truth that "discovery:get-nodes" reads from. -// All mutations come from the scanner-event reader goroutine; reads come -// from RPC handlers, so the lock is sized for a low-contention r/w mix. +// Scanner, manual-worker, trust, and restart updates are serialized by the +// broker's discoveryMu; RPC readers use the store's own read lock. // // onChange, when non-nil, is invoked after every Upsert and Remove // AFTER the write lock has been released. It's the broker's hook for @@ -259,9 +259,8 @@ func (sn storedNode) hasSource(source nodeSource) bool { // off the lock means a caller is free to snapshot inside it without // deadlocking. The store does not guarantee that two rapid mutations // produce two separate callback invocations in strict happens-before -// order with the lock — but since the only writer in practice is the -// single-threaded scanner-event reader, that ordering already holds at -// the call site. +// order with the lock. The broker orders mutations and callbacks at the call +// sites with discoveryMu. type discoveryStore struct { mu sync.RWMutex nodes map[string]storedNode @@ -314,8 +313,8 @@ func (s *discoveryStore) SetOnChange(fn func()) { } // Upsert records one source's view of a node (node/discovered or node/updated). -// The record is keyed by hostUuid and merges the source's payload with any -// other source already claiming that key. The client snapshot is always the +// Verified sources share a host UUID key; unverified discoveries keep their +// separate record keys. The client snapshot is always the // latest full picture, not a diff stream. func (s *discoveryStore) Upsert(n EnrichedNode, source nodeSource) { key := n.storeKey() @@ -466,7 +465,7 @@ func (sn storedNode) toAvailable() AvailableNode { // (id-correlation + read pump), so the broker can Call the scanner and push // notifications down to it — the foundation for relaying discovery:register / // subscribe to the promoted daemon and reading its acks — while -// the scanner's node/* notifications are routed into the discovery store. +// the scanner's discovery:node-* notifications enter the discovery store. type scannerProcess struct { stateMu *sync.Mutex mesh *clustertrust.Mesh @@ -626,10 +625,9 @@ func (s *scannerProcess) reloadIdentity() { } } -// reloadTrust tells the daemon this node's cluster pin set changed, so it -// re-derives the trusted annotation on every node in its directory. The daemon -// otherwise only answers that question when a peer's mDNS record moves, and a -// peer that was already advertising when this node joined never moves again. +// reloadTrust promptly invalidates stale peer proofs and schedules candidate +// authentication after a pin change. Scanner also reconciles live trust on its +// periodic refresh loops, independently of mDNS changes. func (s *scannerProcess) reloadTrust() { ctx, cancel := context.WithTimeout(context.Background(), pushRegisterTimeout) defer cancel() diff --git a/services/nvpair-ui-broker/nodeinfo.go b/services/nvpair-ui-broker/nodeinfo.go index eb083954a..7a3febc23 100644 --- a/services/nvpair-ui-broker/nodeinfo.go +++ b/services/nvpair-ui-broker/nodeinfo.go @@ -212,9 +212,8 @@ func startNodeInfo(binaryPath, logLevel string, onNotify func(method string, par done: make(chan struct{}), } - // Read stdout rather than discarding it: node-info reports the local - // addresses peers have reached it on, and an unattended read end would also - // risk wedging the child on a full pipe buffer. + // Read owned hardware snapshots and reached-address reports from stdout. + // An unread pipe would eventually block the child's notifications. go drainNodeInfoStdout(stdout, onNotify) go func() { _ = cmd.Wait() @@ -230,8 +229,8 @@ func startNodeInfo(binaryPath, logLevel string, onNotify func(method string, par // as is anything that isn't JSON. // // This must never stop reading while the child lives: node-info writes its -// observed-address reports here, and an unread stdout pipe fills up and blocks -// the child mid-write, which silently ends address reporting for the rest of the +// inventory and address reports here, and an unread stdout pipe blocks +// the child mid-write, which silently ends both notification streams for the // session. bufio.Scanner cannot be used for that reason — it fails permanently on // a frame past its limit — so an oversized frame is dropped on its own and the // next one is read normally. @@ -286,7 +285,7 @@ func handleNodeInfoFrame(line []byte, onNotify func(method string, params json.R // logNodeInfoDrainEnd separates the expected end of the stream — the child // exited or Stop closed the pipe — from a read failure, which ends -// observed-address delivery and is worth a warning. +// owned inventory and address delivery and is worth a warning. func logNodeInfoDrainEnd(err error) { if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) || errors.Is(err, io.ErrClosedPipe) { slog.Debug("node-info stdout closed", "err", err) diff --git a/services/nvpair-workload-manager/relaysource.go b/services/nvpair-workload-manager/relaysource.go index 1de97d716..89a1e479d 100644 --- a/services/nvpair-workload-manager/relaysource.go +++ b/services/nvpair-workload-manager/relaysource.go @@ -54,9 +54,9 @@ func (r *relayPeerSource) Nodes(context.Context) ([]PeerNode, error) { return out, nil } -// set replaces the relay peer set from a discovery:nodes snapshot: it projects -// every node advertising wl with a dialable IP (dropping the rest) and applies -// the self-filter. The broker sends the full filtered set on every change, so +// set replaces the relay peer set from a discovery:nodes snapshot, keeping +// trusted pinned peers with a workload endpoint and excluding this node. +// The broker sends the full filtered set on every change, so // this is a wholesale replace, not a per-node apply — a departed peer is simply // absent from the next snapshot. func (r *relayPeerSource) set(nodes []noderec.DirectoryNode) { @@ -78,9 +78,9 @@ func (r *relayPeerSource) set(nodes []noderec.DirectoryNode) { r.mu.Unlock() } -// directoryToPeerNode projects a relay DirectoryNode onto a PeerNode for the wl -// service, returning false when the node doesn't advertise wl or has no dialable -// address. ID is the stable hostUuid — peerSet.Replace keys the broadcast set by +// directoryToPeerNode projects a trusted pinned record with a workload service +// and dialable address onto a PeerNode. ID is the stable hostUuid — +// peerSet.Replace keys the broadcast set by // PeerNode.ID, so two peers sharing a hostname must key on distinct UUIDs or // they'd collapse into one and one would silently miss every broadcast. // Host stays the hostname (display / dial fallback); the peer is diff --git a/services/shared/clustertrust/mesh.go b/services/shared/clustertrust/mesh.go index 287b681fe..1651b9de3 100644 --- a/services/shared/clustertrust/mesh.go +++ b/services/shared/clustertrust/mesh.go @@ -19,10 +19,9 @@ import ( // ClusterUUIDTXTKey is the mDNS TXT key carrying a node's cluster principal — // the UUID of the mTLS leaf nvpair-cluster-manager issued it — so a peer can map -// an advertised service to the pin it holds for that node. It is deliberately -// distinct from node-info's existing uuid= key (the stable per-host node id): -// the cluster principal is a per-cluster cert UUID, not the host UUID, and the -// two are not equal. A service advertises this key only while clustered. +// an advertised service to the pin it holds for that node. Authenticated +// inventory requires the host UUID, this principal, and the serving certificate +// UUID to match. A service advertises this key only while clustered. const ClusterUUIDTXTKey = "cluster-uuid" // ErrNotClustered is returned by the TLS config resolvers while this node is not From 90168285a6152e203d204ca07d6c9e3f9bffb961 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Mon, 26 Oct 2026 08:27:53 -0400 Subject: [PATCH 8/8] Simplify discovery state and shared test fixtures - Replace three UUID-keyed local model maps with one localModels value under the existing peer lock. Flat, per-engine, and loaded inventory now stay together when local services are republished or the host UUID changes. - Remove dropSelf's replacement-UUID argument and local timestamp migration. Local hardware now arrives through owned IPC, so it no longer needs the remote polling cache's identity-migration bookkeeping. - Remove the scanner's second saved-certificate map and use the proof already stored in each verified directory record. That proof binds the current pin and admission, avoiding two collections that must be invalidated together. Keep exact-certificate validation for fresh responses and test that a readmitted peer must authenticate again before its telemetry is renewed. - Capture one copied model-refresh target before waiting for a worker slot. Use its proof, addresses, and endpoint through the request and result checks, replacing tuple arguments, the secondary proof lookup, and duplicate helper plumbing. Keep the missing-target preflight and atomic endpoint guard. - Add a queued-readmission regression with an unchanged certificate and endpoint. It proves that the simpler refresh path rejects the queued old proof and accepts a newly authenticated target. - Remove the broker hardware-snapshot cache that was read only to forward the same event immediately. Validate and forward the event directly, keeping record/principal checks, elapsed sample age, owned IPC replay, and compact scheduling telemetry without cache storage and invalidation paths. - Check hardware forwarding at the client-visible notification boundary. Replaced-record, missing/conflicting identity, and late-snapshot cases now prove what can reach clients instead of inspecting the deleted cache. - Remove the unused manual-notification wrapper. Drive fixtures through the generation-aware callback used by the supervised worker so tests exercise the same retired-worker guard as production. - Reuse shared GPU, CPU, memory, and NodeInfo types in manual probes. Keep only the private proof and receipt time alongside the shared snapshot, and use its aging helper directly so duplicate wire schemas cannot drift. - Derive peer-metadata presence from its accepted nonempty host identity. Remove the separate known flag, which could otherwise disagree with the metadata value, and cover missing-host input explicitly. - Centralize fixture pin writing in WritePeerPinFromCertificate. Scanner and node-info tests reuse the same exact-leaf JSON setup while rotation helpers still generate fresh certificates to test replacement. - Combine overlapping local model-publication tests into the blocked-refresh scenario. It now checks flat, per-engine, and loaded inventories together, keeping the synchronization that proves publication does not wait on HTTP. - Fold the duplicate blocked-metadata test into the existing coalescing test. One fixture now proves both that registry updates continue during a blocked write and that the newest queued metadata is ultimately delivered. - Fold the standalone proxy resubscription test into the multi-engine subscription scenario. Retain assertions that one engine replaces its own subscription and leaves the other engine's current snapshot intact. - Fold the duplicate TUI TLS-hardware check into the existing node-update test. Keep both rendered cached hardware and the no-plaintext-poll assertion without building a second detail-screen fixture. - Remove public-hardware setup from manual engine-only fixtures and rename the helper to configureHealthyOllama. Those tests concern engine routes, while separate public-rejection and authenticated-hardware tests cover measurement acceptance. - Remove an unreachable scanner HTTP fixture. The owned-local path no longer has a public HTTP client that could reach it, so the fixture added setup without observing a possible request. - Share the authenticated peer-readiness poller between model and workload integration tests. Put identity fields in the common available-node type and keep model-specific conditions in the caller, avoiding duplicate JSON wrappers and allowing only the intended pinned peer to satisfy readiness. - Use a stoppable readiness timer and retain bounded polling. The shared helper releases timer resources on an early success while preserving the same failure deadline. - Give the first manual-route integration fixture a separate thirty-second worker-startup allowance. Observed fresh Windows launches nearly exhausted its old ten-second limit before routing convergence was even checked. - Replace obsolete TCP-eviction narratives with the current authenticated inventory and inference-activity contract. Update forwarding and model-write comments to name the remaining paths, so the documentation describes the code reviewers actually need to maintain. - Validation after rebase: full shared, node-info, scanner, manual-nodes, broker, and TUI Go suites; all eight controlled cross-process regressions; service contracts; committed Go formatting; changed-file SPDX headers; and whitespace checks pass. Desktop static/unit checks and Windows service builds were validated earlier, and their sources are unchanged by rebase. Signed-off-by: Kaylee Lubick --- services/nvpair-manual-nodes/manager.go | 48 +--- services/nvpair-manual-nodes/manager_test.go | 63 ++---- .../nvpair-node-info/cluster_mtls_test.go | 15 +- services/nvpair-node-info/main.go | 5 +- services/nvpair-node-info/main_test.go | 11 +- .../cluster_membership_test.go | 17 +- .../nvpair-node-scanner/connreuse_test.go | 6 +- services/nvpair-node-scanner/daemon.go | 212 +++++++----------- services/nvpair-node-scanner/daemon_test.go | 136 ++++++++--- services/nvpair-node-scanner/directory.go | 2 +- services/nvpair-node-scanner/refresh_test.go | 145 +++++------- services/nvpair-tui/ui/nodesmodel_test.go | 17 +- services/nvpair-ui-broker/README.md | 2 +- services/nvpair-ui-broker/broker.go | 21 +- services/nvpair-ui-broker/broker_test.go | 17 +- services/nvpair-ui-broker/discovery.go | 42 +--- services/nvpair-ui-broker/discovery_test.go | 103 ++++++--- services/nvpair-ui-broker/manualnodes_test.go | 5 +- services/nvpair-ui-broker/nodeinfo_test.go | 80 +++---- .../nvpair-ui-broker/proxyaddressing_test.go | 64 ++---- .../clustertrusttest/clustertrusttest.go | 7 + services/shared/discovery/discovery.go | 62 ++--- services/shared/noderec/noderec.go | 8 +- services/tests/broker_management_test.go | 4 +- services/tests/broker_test.go | 21 +- services/tests/models_http_test.go | 4 +- services/tests/models_refresh_test.go | 39 ++-- services/tests/workload_interop_test.go | 38 +--- 28 files changed, 482 insertions(+), 712 deletions(-) diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index b0740f5ff..98f909460 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -46,44 +46,18 @@ const ( probeFailThreshold = 3 ) -type GPUInfo struct { - Name string `json:"name"` - VramBytes uint64 `json:"vram_bytes,omitempty"` - VramUsedBytes uint64 `json:"vram_used_bytes,omitempty"` - UtilizationPercent uint32 `json:"utilization_percent,omitempty"` -} - -// CPUInfo and MemoryInfo mirror the top-level objects nvpair-node-info -// now emits. Pointer-optional so manually-added nodes that aren't -// actually running nvpair-node-info (Ollama-only routing targets) don't -// falsely surface as having an unknown CPU + zero memory — the whole -// object stays absent until we see one in a real response. -type CPUInfo struct { - Name string `json:"name,omitempty"` - Cores uint32 `json:"cores,omitempty"` - UtilizationPercent uint32 `json:"utilization_percent,omitempty"` -} - -type MemoryInfo struct { - TotalBytes uint64 `json:"total_bytes,omitempty"` - UsedBytes uint64 `json:"used_bytes,omitempty"` -} +type ( + GPUInfo = noderec.GPUInfo + CPUInfo = noderec.CPUInfo + MemoryInfo = noderec.MemoryInfo +) +// NodeInfoResponse adds the local proof and receipt time to the shared snapshot. +// Those fields never travel over the hardware response's JSON boundary. type NodeInfoResponse struct { + noderec.NodeInfo discoveryRecordID string receivedAt time.Time - GPUs []GPUInfo `json:"GPUs"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` - // HostUUID is the remote node's stable per-host identity, reported by - // node-info. We surface it so a manual node is tracked by the same permanent - // identity as the rest of the fleet — including deduping with the same - // machine if it's also discovered over mDNS. Empty when the remote - // predates this field or isn't a NVPAIR node-info server. - HostUUID string `json:"hostUuid,omitempty"` - ClusterUUID *string `json:"clusterUuid,omitempty"` } // ManualEntry identifies a user-configured endpoint by its bare hostname or IP @@ -660,11 +634,7 @@ func (m *Manager) probeNodeInfo(client *http.Client, addr string, port int, loca if recordID == "" || recordID != m.mesh.PeerRecordID(result.HostUUID) { return false, NodeInfoResponse{} } - aged := (noderec.NodeInfo{ - TelemetryValid: result.TelemetryValid, - MSSince: result.MSSince, - }).Aged(time.Since(start)) - result.MSSince = aged.MSSince + result.NodeInfo = result.NodeInfo.Aged(time.Since(start)) result.discoveryRecordID = recordID result.receivedAt = time.Now() slog.Debug("manual probe node-info up", diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index 0aab40c5b..d15a0c31f 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -97,11 +97,7 @@ func TestProbeNode_PublicHardwareCannotSupplyServiceMeasurements(t *testing.T) { const modelName = "model-a" m, _, transport := newTestManager() entry := ManualEntry{Address: address, Name: "manual"} - configureHealthyNode(t, transport, entry.Address, []string{modelName}, NodeInfoResponse{ - HostUUID: "victim", - TelemetryValid: true, - GPUs: []GPUInfo{{UtilizationPercent: 1}}, - }) + configureHealthyOllama(t, transport, entry.Address, []string{modelName}) var hardwareReads int transport.set(http.MethodGet, address+":14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { hardwareReads++ @@ -119,7 +115,7 @@ func TestProbeNode_PublicHardwareCannotSupplyServiceMeasurements(t *testing.T) { assert.Equal(t, []string{modelName}, status.OllamaModels) } -func configureHealthyNode(t *testing.T, rt *fakeRoundTripper, addr string, models []string, info NodeInfoResponse) { +func configureHealthyOllama(t *testing.T, rt *fakeRoundTripper, addr string, models []string) { t.Helper() host := net.JoinHostPort(addr, "11434") rt.set(http.MethodGet, host, "/", func(*http.Request) (*http.Response, error) { @@ -140,13 +136,6 @@ func configureHealthyNode(t *testing.T, rt *fakeRoundTripper, addr string, model assert.NoError(t, err, "marshal model inventory") return httpJSON(http.StatusOK, string(data)) }) - - infoHost := net.JoinHostPort(addr, "14318") - rt.set(http.MethodGet, infoHost, "/v1/node-info", func(*http.Request) (*http.Response, error) { - data, err := json.Marshal(info) - assert.NoError(t, err, "marshal node info") - return httpJSON(http.StatusOK, string(data)) - }) } // configureHealthyLMStudio registers a 200 GET /v1/models on addr:1234 @@ -295,16 +284,18 @@ func methodIs(method string) func(Message) bool { func sampleInfo() NodeInfoResponse { return NodeInfoResponse{ - GPUs: []GPUInfo{{ - Name: "RTX 6000", - VramBytes: 48 << 30, - VramUsedBytes: 12 << 30, - UtilizationPercent: 42, - }}, - CPU: &CPUInfo{Name: "Threadripper", Cores: 64, UtilizationPercent: 7}, - Memory: &MemoryInfo{TotalBytes: 128 << 30, UsedBytes: 32 << 30}, - TelemetryValid: true, - MSSince: 137, + NodeInfo: noderec.NodeInfo{ + GPUs: []GPUInfo{{ + Name: "RTX 6000", + VramBytes: 48 << 30, + VramUsedBytes: 12 << 30, + UtilizationPercent: 42, + }}, + CPU: &CPUInfo{Name: "Threadripper", Cores: 64, UtilizationPercent: 7}, + Memory: &MemoryInfo{TotalBytes: 128 << 30, UsedBytes: 32 << 30}, + TelemetryValid: true, + MSSince: 137, + }, } } @@ -315,7 +306,7 @@ func TestNodeID_PrefersConfiguredNameOverAddressFallback(t *testing.T) { func TestNodeAdd_RespondsWithInitialStatusAndDiscoversProbeResult(t *testing.T) { m, rw, rt := newTestManager() - configureHealthyNode(t, rt, "node.local", []string{"llama3", "mistral"}, sampleInfo()) + configureHealthyOllama(t, rt, "node.local", []string{"llama3", "mistral"}) m.handleMessage(requestMessage(t, 1, "node/add", ManualEntry{Address: "node.local", Name: "lab"})) @@ -364,7 +355,7 @@ func TestNodeAdd_RejectsInvalidParameters(t *testing.T) { func TestNodesList_ReturnsCurrentStatuses(t *testing.T) { m, rw, rt := newTestManager() - configureHealthyNode(t, rt, "node.local", []string{"llama3"}, sampleInfo()) + configureHealthyOllama(t, rt, "node.local", []string{"llama3"}) m.handleMessage(requestMessage(t, 1, "node/add", ManualEntry{Address: "node.local"})) _ = readCaptureUntil(t, rw, methodIs("node/discovered")) @@ -416,9 +407,6 @@ func TestAddThenRemoveBeforeInitialProbe_DoesNotRediscover(t *testing.T) { rt.set(http.MethodGet, net.JoinHostPort("node.local", "11434"), "/api/tags", func(*http.Request) (*http.Response, error) { return httpJSON(http.StatusOK, `{"models":[]}`) }) - rt.set(http.MethodGet, net.JoinHostPort("node.local", "14318"), "/v1/node-info", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{"GPUs":[]}`) - }) status := m.addNode(ManualEntry{Address: "node.local", Name: "lab"}) assert.Equal(t, "lab", status.ID, "add status (%v)", status) @@ -441,12 +429,12 @@ func TestProbeNode_EmitsUpdatedOnStateChange(t *testing.T) { m, rw, rt := newTestManager() m.nodes["lab"] = &trackedNode{entry: ManualEntry{Name: "lab", Address: "node.local"}, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}} - configureHealthyNode(t, rt, "node.local", []string{"llama3"}, sampleInfo()) + configureHealthyOllama(t, rt, "node.local", []string{"llama3"}) m.probeNode(ManualEntry{Name: "lab", Address: "node.local"}) first := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) assert.Equal(t, []string{"llama3"}, first.OllamaModels, "first update models") - configureHealthyNode(t, rt, "node.local", []string{"mistral"}, sampleInfo()) + configureHealthyOllama(t, rt, "node.local", []string{"mistral"}) m.probeNode(ManualEntry{Name: "lab", Address: "node.local"}) second := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) assert.Equal(t, []string{"mistral"}, second.OllamaModels, "second update models") @@ -456,7 +444,7 @@ func TestProbeNode_EmitsNothingWhenStateIsUnchanged(t *testing.T) { m, rw, rt := newTestManager() entry := ManualEntry{Name: "lab", Address: "node.local"} m.nodes["lab"] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}} - configureHealthyNode(t, rt, "node.local", []string{"llama3"}, sampleInfo()) + configureHealthyOllama(t, rt, "node.local", []string{"llama3"}) m.probeNode(entry) _ = readCaptureUntil(t, rw, methodIs("node/updated")) @@ -555,14 +543,7 @@ func newManualPeerFixture(t *testing.T) *manualPeerFixture { t.Helper() certificate, err := os.ReadFile(filepath.Join(source, "node.crt")) require.NoError(t, err) - type peerPin struct { - NodeUUID string `json:"nodeUuid"` - CertPEM string `json:"certPem"` - } - data, err := json.Marshal(peerPin{NodeUUID: uuid, CertPEM: string(certificate)}) - require.NoError(t, err) - require.NoError(t, os.MkdirAll(filepath.Join(destination, "trusted"), 0o700)) - require.NoError(t, os.WriteFile(filepath.Join(destination, "trusted", uuid+".json"), data, 0o600)) + clustertrusttest.WritePeerPinFromCertificate(t, destination, uuid, certificate) } pin(clientDir, serverDir, manualPeerUUID) pin(serverDir, clientDir, selfUUID) @@ -778,7 +759,7 @@ func TestProbeNode_DiscardsOldIdentityAfterHostnameReassignment(t *testing.T) { } f.entry.Address = "manual.example" f.manager.nodes[nodeID(f.entry)].entry = f.entry - configureHealthyNode(t, engineTransport, f.entry.Address, []string{"original-model"}, NodeInfoResponse{}) + configureHealthyOllama(t, engineTransport, f.entry.Address, []string{"original-model"}) f.manager.probeNode(f.entry) warmed := f.manager.listNodes() require.Len(t, warmed, 1) @@ -794,7 +775,7 @@ func TestProbeNode_DiscardsOldIdentityAfterHostnameReassignment(t *testing.T) { targetMu.Lock() target = replacement.Listener.Addr().String() targetMu.Unlock() - configureHealthyNode(t, engineTransport, f.entry.Address, []string{"replacement-model"}, NodeInfoResponse{}) + configureHealthyOllama(t, engineTransport, f.entry.Address, []string{"replacement-model"}) f.manager.probeNode(f.entry) diff --git a/services/nvpair-node-info/cluster_mtls_test.go b/services/nvpair-node-info/cluster_mtls_test.go index 4cf154ff3..d62ea5f47 100644 --- a/services/nvpair-node-info/cluster_mtls_test.go +++ b/services/nvpair-node-info/cluster_mtls_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" ) @@ -63,16 +64,8 @@ func setupNode(t *testing.T, certPEM, keyPEM []byte, pins map[string][]byte) str dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "node.crt"), certPEM, 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "node.key"), keyPEM, 0o600)) - td := filepath.Join(dir, "trusted") - require.NoError(t, os.MkdirAll(td, 0o700)) - type peerPin struct { - NodeUUID string `json:"nodeUuid"` - CertPEM string `json:"certPem"` - } for uuid, pcert := range pins { - body, err := json.Marshal(peerPin{NodeUUID: uuid, CertPEM: string(pcert)}) - require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(td, uuid+".json"), body, 0o600)) + clustertrusttest.WritePeerPinFromCertificate(t, dir, uuid, pcert) } return dir } @@ -97,7 +90,7 @@ func TestNodeInfoHandler_RequiresPinnedClients(t *testing.T) { principal := "uuid-a" wantInfo := buildResponse(nil, nil, 0, statsSnapshot{}, principal, &principal) - metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318), known: true} + metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318)} mux := http.NewServeMux() mux.HandleFunc("/v1/node-info", nodeInfoHandler(meshA, metadata, func() NodeInfoResponse { return wantInfo })) srv := httptest.NewUnstartedServer(mux) @@ -155,7 +148,7 @@ func TestNodeInfoHandler_PreservesHardwareForPinnedPeers(t *testing.T) { TelemetryValid: true, MSSince: 137, HostUUID: principal, ClusterUUID: &principal, } - metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318), known: true} + metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318)} server := httptest.NewUnstartedServer(nodeInfoHandler(serverMesh, metadata, func() NodeInfoResponse { return snapshot })) server.TLS = serverMesh.ServerTLSConfig() server.StartTLS() diff --git a/services/nvpair-node-info/main.go b/services/nvpair-node-info/main.go index b188b063e..b7af9cc70 100644 --- a/services/nvpair-node-info/main.go +++ b/services/nvpair-node-info/main.go @@ -150,14 +150,13 @@ type peerMetadata struct { // Readers must see a complete snapshot rather than partially replaced fields. mu sync.RWMutex value noderec.PeerMetadataParams - known bool } // get returns the latest registry snapshot and whether the broker supplied one. func (m *peerMetadata) get() (noderec.PeerMetadataParams, bool) { m.mu.RLock() defer m.mu.RUnlock() - return m.value, m.known + return m.value, m.value.HostUUID != "" } // handlePeerMetadata replaces registry metadata from the broker, rejecting @@ -172,7 +171,7 @@ func handlePeerMetadata(msg applog.StdinMessage, hostUUID string, metadata *peer valid := err == nil && hostUUID != "" && params.HostUUID == hostUUID if valid { metadata.mu.Lock() - metadata.value, metadata.known = params, true + metadata.value = params metadata.mu.Unlock() } if len(msg.ID) == 0 { diff --git a/services/nvpair-node-info/main_test.go b/services/nvpair-node-info/main_test.go index efc155a43..9b84b6b1e 100644 --- a/services/nvpair-node-info/main_test.go +++ b/services/nvpair-node-info/main_test.go @@ -301,6 +301,7 @@ func TestPeerMetadata_RejectsMalformedOrForeignIdentity(t *testing.T) { }) } test("malformed object", `"invalid"`) + test("missing host", `{"peer":{"name":"copied-name"}}`) test("foreign host", `{"hostUuid":"someone-else","peer":{"name":"copied-name"}}`) } @@ -453,13 +454,7 @@ func TestNodeInfo_RechecksTrustOnReusedConnections(t *testing.T) { test("pin replacement with the same UUID", func(t *testing.T, dir string) { t.Helper() cert, _ := genLeaf(t, "uuid-b") - type trustedPeer struct { - NodeUUID string `json:"nodeUuid"` - CertPEM string `json:"certPem"` - } - data, err := json.Marshal(trustedPeer{NodeUUID: "uuid-b", CertPEM: string(cert)}) - require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(dir, "trusted", "uuid-b.json"), data, 0o600)) + clustertrusttest.WritePeerPinFromCertificate(t, dir, "uuid-b", cert) }) test("local departure retaining the keypair", func(t *testing.T, dir string) { t.Helper() @@ -480,7 +475,7 @@ func TestNodeInfoHandler_RejectsInventoryIfTrustChangesDuringCollection(t *testi require.True(t, ok, "client certificate fixture must be available") require.NotEmpty(t, clientConfig.Certificates) require.NotNil(t, clientConfig.Certificates[0].Leaf) - metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318), known: true} + metadata := &peerMetadata{value: nodeInfoMetadataForTest(14318)} // Call the handler synchronously with the authenticated client's TLS state // so trust can be changed deterministically while it builds the response. request := httptest.NewRequest(http.MethodGet, "https://localhost/v1/node-info", nil) diff --git a/services/nvpair-node-scanner/cluster_membership_test.go b/services/nvpair-node-scanner/cluster_membership_test.go index 28188d20a..e7f5e02ae 100644 --- a/services/nvpair-node-scanner/cluster_membership_test.go +++ b/services/nvpair-node-scanner/cluster_membership_test.go @@ -19,16 +19,13 @@ import ( func newMembershipTestDaemon(clusterDir string) *daemon { mesh := clustertrust.Open(clusterDir) return &daemon{ - mesh: mesh, - modelsClients: clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout), - clusterDir: clusterDir, - baseDir: filepath.Dir(clusterDir), - reg: newRegistry("host-uuid", "", selfAddrs("127.0.0.1")), - dir: newDirectory(), - lastInfoAt: make(map[string]time.Time), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), + mesh: mesh, + modelsClients: clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout), + clusterDir: clusterDir, + baseDir: filepath.Dir(clusterDir), + reg: newRegistry("host-uuid", "", selfAddrs("127.0.0.1")), + dir: newDirectory(), + lastInfoAt: make(map[string]time.Time), // codec/responder nil: emit is nil-safe; reloadIdentity skips re-advertise. } } diff --git a/services/nvpair-node-scanner/connreuse_test.go b/services/nvpair-node-scanner/connreuse_test.go index 02b678a6c..a62b4d12a 100644 --- a/services/nvpair-node-scanner/connreuse_test.go +++ b/services/nvpair-node-scanner/connreuse_test.go @@ -86,9 +86,5 @@ func writePinFromCert(t *testing.T, clusterDir, peerUUID, certPath string) { t.Helper() certPEM, err := os.ReadFile(certPath) require.NoError(t, err) - trusted := filepath.Join(clusterDir, "trusted") - require.NoError(t, os.MkdirAll(trusted, 0o700)) - body, err := json.Marshal(map[string]string{"nodeUuid": peerUUID, "certPem": string(certPEM)}) - require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(trusted, peerUUID+".json"), body, 0o600)) + clustertrusttest.WritePeerPinFromCertificate(t, clusterDir, peerUUID, certPEM) } diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 23268d138..633583cf7 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -56,11 +56,10 @@ type daemon struct { nodeInfoClients *clustertrust.PeerClientPool // peerMu serializes candidate generations, proof invalidation, and accepted // results. No network request runs while it is held. - peerMu sync.Mutex - candidates map[string]*discoveryCandidate - peerCertificates map[string]*x509.Certificate - authWake chan struct{} - authRetries telemetryRetryGate + peerMu sync.Mutex + candidates map[string]*discoveryCandidate + authWake chan struct{} + authRetries telemetryRetryGate // nodeInfoOrigins queues verified same-origin reads before http.Client starts // its timeout. Candidate authentication skips busy origins and retries later. @@ -89,19 +88,17 @@ type daemon struct { // localInfo is supplied only by the owned worker IPC path, under selfMu. localInfo *noderec.NodeInfo - // infoMu guards peer read timestamps, model inventories, and outage state. + // infoMu guards peer read timestamps and outage state. infoMu sync.Mutex // lastInfoAt records accepted node-info reads for authenticated peer liveness. lastInfoAt map[string]time.Time - // These model caches preserve owned-local inventory when publication rebuilds - // the local record during an outage. Remote records retain models in the directory. - lastModels map[string][]string - lastModelsByEngine map[string]map[string][]string - lastLoadedByEngine map[string]map[string][]string + // localModels preserves owned inventory when publication rebuilds the local + // record, including after its UUID changes. Guarded by peerMu. Remote models + // stay in the directory. + localModels modelInventory // nodeInfoDown holds the nodes whose last node-info enrichment failed, so // the outage is reported when it starts and when it ends rather than once - // per sweep. Guarded by infoMu with the caches it explains: the reason a - // card keeps showing an old inventory is exactly this entry. + // per sweep. Guarded by infoMu; the directory retains the last accepted inventory. nodeInfoDown map[string]bool // activityMu guards lastActivityAt, when each peer was last seen returning @@ -319,26 +316,22 @@ func newDaemon(codec *Codec, mesh *clustertrust.Mesh, clusterDir string) *daemon instance, _ := os.Hostname() d := &daemon{ - codec: codec, - mesh: mesh, - reg: reg, - instance: instance, - baseDir: baseDir, - clusterDir: clusterDir, - dir: newDirectory(), - modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, - modelsClients: clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout), - modelsWake: make(chan struct{}, 1), - nodeInfoClients: newNodeInfoClients(mesh), - candidates: make(map[string]*discoveryCandidate), - peerCertificates: make(map[string]*x509.Certificate), - authWake: make(chan struct{}, 1), - lastInfoAt: make(map[string]time.Time), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), - nodeInfoDown: make(map[string]bool), - lastActivityAt: make(map[string]time.Time), + codec: codec, + mesh: mesh, + reg: reg, + instance: instance, + baseDir: baseDir, + clusterDir: clusterDir, + dir: newDirectory(), + modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, + modelsClients: clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout), + modelsWake: make(chan struct{}, 1), + nodeInfoClients: newNodeInfoClients(mesh), + candidates: make(map[string]*discoveryCandidate), + authWake: make(chan struct{}, 1), + lastInfoAt: make(map[string]time.Time), + nodeInfoDown: make(map[string]bool), + lastActivityAt: make(map[string]time.Time), } // Track connection candidates independently of identity claims. Promoted // peers survive multicast gaps using authenticated reads or inference. @@ -599,6 +592,13 @@ func (d *daemon) currentPin(sample peerInfoSample) bool { return sample.recordID != "" && sample.recordID == d.mesh.PeerRecordID(sample.info.HostUUID) && d.mesh.VerifyPeerCertificate(sample.info.HostUUID, sample.certificate) } +// currentPeerRecord checks the proof binding stored by an authenticated read. +// The record ID already includes the exact certificate and admission generation. +// The caller must refresh mesh before checking it. +func (d *daemon) currentPeerRecord(node noderec.DirectoryNode) bool { + return node.IdentityVerification == noderec.IdentityPinned && node.DiscoveryRecordID != "" && node.DiscoveryRecordID == d.mesh.PeerRecordID(node.ClusterUUID) +} + // currentPeerResult also rejects results for an endpoint replaced while a read // was queued or in flight. peerMu must be held through the subsequent write. func (d *daemon) currentPeerResult(target noderec.DirectoryNode, sample peerInfoSample) bool { @@ -606,7 +606,7 @@ func (d *daemon) currentPeerResult(target noderec.DirectoryNode, sample peerInfo return false } node, ok := d.dir.get(target.HostUUID) - return ok && node.IdentityVerification == noderec.IdentityPinned && node.DiscoveryRecordID == target.DiscoveryRecordID && node.ClusterUUID == target.ClusterUUID && node.Services[noderec.ServiceNodeInfo].Port == target.Services[noderec.ServiceNodeInfo].Port && slices.Equal(node.CandidateIPs(), target.CandidateIPs()) && d.mesh.VerifyPeerCertificate(node.ClusterUUID, d.peerCertificates[node.HostUUID]) + return ok && node.IdentityVerification == noderec.IdentityPinned && node.DiscoveryRecordID == sample.recordID && node.DiscoveryRecordID == target.DiscoveryRecordID && node.ClusterUUID == target.ClusterUUID && node.Services[noderec.ServiceNodeInfo].Port == target.Services[noderec.ServiceNodeInfo].Port && slices.Equal(node.CandidateIPs(), target.CandidateIPs()) } func (d *daemon) recordPeerInfoSuccess(uuid string) { @@ -633,10 +633,6 @@ func (d *daemon) publishPeerInfo(candidateID string, sample peerInfoSample) { Name: info.Peer.Name, IP: info.Peer.Addresses[0], IPs: info.Peer.Addresses, Services: services, GPUs: info.GPUs, CPU: info.CPU, Memory: info.Memory, } - if d.peerCertificates == nil { - d.peerCertificates = make(map[string]*x509.Certificate) - } - d.peerCertificates[info.HostUUID] = sample.certificate d.recordPeerInfoSuccess(info.HostUUID) removed, discovered, changed := d.dir.promote(candidateID, node) if removed.DiscoveryRecordID != "" { @@ -718,7 +714,6 @@ func (d *daemon) onBrowse(ev DiscoveryEvent) { if peer, present := d.dir.get(candidate.peerUUID); present { d.forget(peer.HostUUID) d.dir.remove(peer.HostUUID) - delete(d.peerCertificates, peer.HostUUID) d.emit(noderec.NotifyNodeRemoved, peer) } } @@ -775,23 +770,19 @@ func (d *daemon) reconcileIdentity() { func (d *daemon) reconcileTrust() { d.peerMu.Lock() defer d.peerMu.Unlock() - for uuid, cert := range d.peerCertificates { - node, present := d.dir.get(uuid) - if present && node.DiscoveryRecordID == d.mesh.PeerRecordID(uuid) && d.mesh.VerifyPeerCertificate(uuid, cert) { + for _, node := range d.dir.snapshot("") { + if node.IdentityVerification != noderec.IdentityPinned || d.currentPeerRecord(node) { continue } - delete(d.peerCertificates, uuid) for _, candidate := range d.candidates { - if candidate.peerUUID == uuid { + if candidate.peerUUID == node.HostUUID { candidate.peerUUID = "" candidate.pending = true } } - if node, ok := d.dir.get(uuid); ok { - d.forget(uuid) - d.dir.remove(uuid) - d.emit(noderec.NotifyNodeRemoved, node) - } + d.forget(node.HostUUID) + d.dir.remove(node.HostUUID) + d.emit(noderec.NotifyNodeRemoved, node) } if d.nodeInfoClients != nil { d.nodeInfoClients.DropUnpinned() @@ -859,11 +850,10 @@ func (d *daemon) reloadIdentity() { d.responder.UpdateTXT(d.reg.txt()) } // A changed hostUuid (fresh-host re-stamp onto the cluster principal) leaves - // our old entry under the stale uuid. Drop it (carrying its enrichment cache - // onto the new uuid) so no ghost is served, then republish self under the new - // identity below. + // our old entry under the stale uuid. Drop it so no ghost is served, then + // republish the owned inventory under the new identity below. if oldHostUUID != "" && oldHostUUID != hostUUID { - d.dropSelf(oldHostUUID, hostUUID) + d.dropSelf(oldHostUUID) } d.publishSelfLocked() } @@ -956,32 +946,14 @@ func (d *daemon) refreshAdvertisedAddressesOnce() { d.publishSelfLocked() } -// dropSelf removes this node's stale directory entry after a uuid re-stamp, -// carries its model caches onto the new uuid, and emits removed(old) so consumers -// reconcile. +// dropSelf removes this node's stale directory entry after a uuid re-stamp and +// emits removed(old) so consumers reconcile. Owned model inventory is independent +// of the directory UUID and remains available to the next publication. // The new entry is (re)published by the caller via publishSelfLocked. Must be // called with selfMu held. -func (d *daemon) dropSelf(oldHostUUID, newHostUUID string) { +func (d *daemon) dropSelf(oldHostUUID string) { node, existed := d.dir.get(oldHostUUID) d.dir.remove(oldHostUUID) - d.infoMu.Lock() - if at, ok := d.lastInfoAt[oldHostUUID]; ok { - d.lastInfoAt[newHostUUID] = at - } - if models, ok := d.lastModels[oldHostUUID]; ok { - d.lastModels[newHostUUID] = models - } - if byEngine, ok := d.lastModelsByEngine[oldHostUUID]; ok { - d.lastModelsByEngine[newHostUUID] = byEngine - } - if loaded, ok := d.lastLoadedByEngine[oldHostUUID]; ok { - d.lastLoadedByEngine[newHostUUID] = loaded - } - delete(d.lastInfoAt, oldHostUUID) - delete(d.lastModels, oldHostUUID) - delete(d.lastModelsByEngine, oldHostUUID) - delete(d.lastLoadedByEngine, oldHostUUID) - d.infoMu.Unlock() if existed { d.emit(noderec.NotifyNodeRemoved, node) } @@ -1045,11 +1017,9 @@ func (d *daemon) publishSelfLocked() { d.peerMu.Lock() defer d.peerMu.Unlock() if node.HasService(noderec.ServiceEngineManager) { - d.infoMu.Lock() - node.Models = d.lastModels[rec.HostUUID] - node.ModelsByEngine = d.lastModelsByEngine[rec.HostUUID] - node.LoadedByEngine = d.lastLoadedByEngine[rec.HostUUID] - d.infoMu.Unlock() + node.Models = d.localModels.models + node.ModelsByEngine = d.localModels.byEngine + node.LoadedByEngine = d.localModels.loadedByEngine } method := noderec.NotifyNodeUpdated @@ -1166,9 +1136,8 @@ func (d *daemon) peerReachable(uuid string) bool { } d.peerMu.Lock() node, ok := d.dir.get(uuid) - cert := d.peerCertificates[uuid] d.mesh.Refresh() - valid := ok && node.DiscoveryRecordID == d.mesh.PeerRecordID(uuid) && d.mesh.VerifyPeerCertificate(node.ClusterUUID, cert) + valid := ok && d.currentPeerRecord(node) d.peerMu.Unlock() if !valid { return false @@ -1458,7 +1427,6 @@ func (d *daemon) expireRetainedPeer(target noderec.DirectoryNode) { } d.forget(peer.HostUUID) d.dir.remove(peer.HostUUID) - delete(d.peerCertificates, peer.HostUUID) d.emit(noderec.NotifyNodeRemoved, peer) } @@ -1469,52 +1437,44 @@ func (d *daemon) expireRetainedPeer(target noderec.DirectoryNode) { func (d *daemon) refreshModelsOnce() { sem := make(chan struct{}, modelsRefreshConcurrency) var wg sync.WaitGroup - for _, n := range d.dir.snapshot(noderec.ServiceEngineManager) { - em, ok := n.Services[noderec.ServiceEngineManager] - if !ok || n.IP == "" || (n.IdentityVerification != noderec.IdentityLocal && n.IdentityVerification != noderec.IdentityPinned) { + for _, target := range d.dir.snapshot(noderec.ServiceEngineManager) { + if !target.HasService(noderec.ServiceEngineManager) || target.IP == "" || (target.IdentityVerification != noderec.IdentityLocal && target.IdentityVerification != noderec.IdentityPinned) { continue } + // Keep the addresses and endpoint from this sweep even while a previous + // worker holds the last permit and newer inventory replaces the directory. + target.IPs = slices.Clone(target.CandidateIPs()) + target.Services = maps.Clone(target.Services) wg.Add(1) sem <- struct{}{} - hosts := append([]string(nil), n.CandidateIPs()...) - go func(hostUUID, guardIP string, hosts []string, port int, clusterUUID string) { + go func(target noderec.DirectoryNode) { defer wg.Done() defer func() { <-sem }() - d.refreshNodeModelsCandidates(hostUUID, guardIP, hosts, port, clusterUUID) - }(n.HostUUID, n.IP, hosts, em.Port, n.ClusterUUID) + d.refreshNodeModels(target) + }(target) } wg.Wait() } -// refreshNodeModels snapshots the complete address list before fetching model -// inventory. The shared result path validates current endpoints and remote proof -// before changing the directory, caches, or notifications. Failed reads preserve -// the last accepted inventory. Local reads use the owned engine-manager's -// loopback HTTP endpoint; remote reads require the verified peer's pinned mTLS. -func (d *daemon) refreshNodeModels(hostUUID, ip string, emPort int, clusterUUID string) bool { - node, present := d.dir.get(hostUUID) - if !present { +// refreshNodeModels fetches inventory for a target captured before queuing. Its +// complete address and proof snapshot guards against queued or in-flight results +// from a replaced endpoint. Local reads use the owned engine-manager's loopback +// HTTP endpoint; remote reads require the verified peer's pinned mTLS. +func (d *daemon) refreshNodeModels(target noderec.DirectoryNode) bool { + if _, present := d.dir.get(target.HostUUID); !present { return false } - return d.refreshNodeModelsCandidates(hostUUID, ip, append([]string(nil), node.CandidateIPs()...), emPort, clusterUUID) -} - -// refreshNodeModelsCandidates is refreshNodeModels with address failover. The -// complete remote address list guards against queued or in-flight results from -// an endpoint removed by a newer authenticated inventory response. -// -// askRemembered retains a working address across model sweeps until it fails or -// is withdrawn from the current record. -func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []string, emPort int, clusterUUID string) bool { - original, _ := d.dir.get(hostUUID) - if d.reg != nil && hostUUID == d.reg.record().HostUUID { + hosts := target.CandidateIPs() + emPort := target.Services[noderec.ServiceEngineManager].Port + clusterUUID := target.ClusterUUID + if d.reg != nil && target.HostUUID == d.reg.record().HostUUID { hosts = []string{loopbackHost} clusterUUID = "" } ask := func(host string) (modelInventory, bool) { return d.fetchModels(host, emPort, clusterUUID) } - key := hostKey{hostUUID: hostUUID, service: noderec.ServiceEngineManager} + key := hostKey{hostUUID: target.HostUUID, service: noderec.ServiceEngineManager} _, inventory, fetched := askRemembered(&d.enrichHosts, key, hosts, ask) if !fetched { return false @@ -1523,35 +1483,21 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s d.peerMu.Lock() defer d.peerMu.Unlock() d.mesh.Refresh() - if target, present := d.dir.get(hostUUID); present && target.IdentityVerification == noderec.IdentityPinned { - if target.ClusterUUID != clusterUUID || - target.DiscoveryRecordID != original.DiscoveryRecordID || - target.DiscoveryRecordID != d.mesh.PeerRecordID(clusterUUID) || - !slices.Equal(target.CandidateIPs(), hosts) || - !d.mesh.VerifyPeerCertificate(clusterUUID, inventory.certificate) || - !d.mesh.VerifyPeerCertificate(clusterUUID, d.peerCertificates[hostUUID]) { + if current, present := d.dir.get(target.HostUUID); present && current.IdentityVerification == noderec.IdentityPinned { + if current.ClusterUUID != clusterUUID || + current.DiscoveryRecordID != target.DiscoveryRecordID || + !d.currentPeerRecord(current) || + !slices.Equal(current.CandidateIPs(), hosts) || + !d.mesh.VerifyPeerCertificate(clusterUUID, inventory.certificate) { return false } } - node, changed, valid := d.dir.applyModels(hostUUID, guardIP, emPort, models, byEngine, loadedByEngine) + node, changed, valid := d.dir.applyModels(target.HostUUID, target.IP, emPort, models, byEngine, loadedByEngine) if !valid { return false } if node.IdentityVerification == noderec.IdentityLocal { - d.infoMu.Lock() - if d.lastModels == nil { - d.lastModels = make(map[string][]string) - } - if d.lastModelsByEngine == nil { - d.lastModelsByEngine = make(map[string]map[string][]string) - } - if d.lastLoadedByEngine == nil { - d.lastLoadedByEngine = make(map[string]map[string][]string) - } - d.lastModels[hostUUID] = models - d.lastModelsByEngine[hostUUID] = byEngine - d.lastLoadedByEngine[hostUUID] = loadedByEngine - d.infoMu.Unlock() + d.localModels = inventory } if changed { diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index 9c12ca511..c6873d834 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -111,18 +111,15 @@ func selfAddrs(ip string) []string { // and no network announcements or IPC notifications are sent. func newSelfTestDaemon(hostUUID, ip string) *daemon { return &daemon{ - mesh: clustertrust.Open(""), - reg: newRegistry(hostUUID, "", selfAddrs(ip)), - instance: "myhost", - dir: newDirectory(), - modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, - lastInfoAt: make(map[string]time.Time), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), - lastActivityAt: make(map[string]time.Time), - authWake: make(chan struct{}, 1), - modelsWake: make(chan struct{}, 1), + mesh: clustertrust.Open(""), + reg: newRegistry(hostUUID, "", selfAddrs(ip)), + instance: "myhost", + dir: newDirectory(), + modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, + lastInfoAt: make(map[string]time.Time), + lastActivityAt: make(map[string]time.Time), + authWake: make(chan struct{}, 1), + modelsWake: make(chan struct{}, 1), } } @@ -130,19 +127,18 @@ func newSelfTestDaemon(hostUUID, ip string) *daemon { // Reloading must use the UUID from that file and remove the old local record. func TestReloadIdentity_UsesClusterIdentityUUIDAndRemovesOldRecord(t *testing.T) { base := t.TempDir() - d := &daemon{ - mesh: clustertrust.Open(""), // No cluster directory means no cluster membership. - reg: newRegistry("scanner-minted-X", "", selfAddrs("127.0.0.1")), - dir: newDirectory(), - baseDir: base, - lastInfoAt: make(map[string]time.Time), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), - // Leave codec and responder nil to suppress notifications and announcements. - } - // The local record initially uses the UUID created by the scanner. - d.dir.upsert(noderec.DirectoryNode{HostUUID: "scanner-minted-X", Name: "host", IP: "127.0.0.1"}) + d := newSelfTestDaemon("scanner-minted-X", "127.0.0.1") + d.baseDir = base + _, port := startModelsStub(t, http.StatusOK, map[string]any{ + "models": []string{"retained-model"}, + "modelsByEngine": map[string][]string{"ollama": {"retained-model"}}, + "loadedByEngine": map[string][]string{"ollama": {}}, + }) + d.reg.register(noderec.RegisterParams{Service: noderec.ServiceEngineManager, Port: port}) + d.publishSelf() + target, present := d.dir.get("scanner-minted-X") + require.True(t, present) + require.True(t, d.refreshNodeModels(target)) // cluster-manager writes the UUID to /cluster/identity.json. idPath := filepath.Join(base, "cluster", "identity.json") @@ -155,8 +151,11 @@ func TestReloadIdentity_UsesClusterIdentityUUIDAndRemovesOldRecord(t *testing.T) assert.Equal(t, "cluster-principal-Y", d.reg.record().HostUUID) // The directory must contain the new UUID and no longer contain the old one. - _, present := d.dir.get("cluster-principal-Y") - assert.True(t, present) + node, present := d.dir.get("cluster-principal-Y") + require.True(t, present) + assert.Equal(t, []string{"retained-model"}, node.Models) + assert.Equal(t, map[string][]string{"ollama": {"retained-model"}}, node.ModelsByEngine) + assert.Equal(t, map[string][]string{"ollama": {}}, node.LoadedByEngine) _, present = d.dir.get("scanner-minted-X") assert.False(t, present) } @@ -328,7 +327,7 @@ func TestPublishSelf_LocalRecordHasLocalVerificationAndDiscoveryRecordID(t *test assert.Equal(t, "local:self-uuid", node.DiscoveryRecordID) } -// Replace the local identity and inspect the old record's removal event. Its +// Remove the old local identity and inspect its removal event. Its // local provenance and discovery record ID must survive so consumers invalidate // the correct source instead of treating it as an unverified disappearance. func TestDropSelf_PreservesRecordIdentityInRemoval(t *testing.T) { @@ -339,7 +338,7 @@ func TestDropSelf_PreservesRecordIdentityInRemoval(t *testing.T) { d.codec = NewCodec(&output) d.selfMu.Lock() - d.dropSelf(oldUUID, "new-self") + d.dropSelf(oldUUID) d.selfMu.Unlock() message, err := NewCodec(&output).Read() @@ -939,6 +938,8 @@ func TestDiscovery_DiscardsModelResponseAfterSavedCertificateReplacement(t *test return true } f.discover(t) + target, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) client, ok := f.daemon.modelsClients.Client("peer-uuid") require.True(t, ok, "authenticated models client must be available") closed, release := make(chan struct{}), make(chan struct{}) @@ -951,7 +952,7 @@ func TestDiscovery_DiscardsModelResponseAfterSavedCertificateReplacement(t *test t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) go func() { defer close(done) - changed <- f.daemon.refreshNodeModels("peer-uuid", "127.0.0.1", port, "peer-uuid") + changed <- f.daemon.refreshNodeModels(target) }() select { case <-closed: @@ -975,7 +976,7 @@ func TestDiscovery_DiscardsModelResponseAfterSavedCertificateReplacement(t *test require.True(t, ok) assert.Equal(t, before, after) assert.Empty(t, output.Bytes()) - require.True(t, f.daemon.refreshNodeModels("peer-uuid", "127.0.0.1", port, "peer-uuid")) + require.True(t, f.daemon.refreshNodeModels(before)) after, _ = f.daemon.dir.get("peer-uuid") assert.Equal(t, response.Models, after.Models) assert.Equal(t, response.ModelsByEngine, after.ModelsByEngine) @@ -1001,7 +1002,9 @@ func TestRefreshNodeModels_AcceptsPinnedPeersWithMultipleAddresses(t *testing.T) } f.discover(t) - assert.True(t, f.daemon.refreshNodeModels(peerUUID, "127.0.0.1", port, peerUUID)) + target, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + assert.True(t, f.daemon.refreshNodeModels(target)) node, present := f.daemon.dir.get(peerUUID) require.True(t, present) @@ -1043,7 +1046,7 @@ func TestDiscovery_DiscardsModelsAfterSecondaryAddressChanges(t *testing.T) { t.Cleanup(func() { releaseOnce.Do(func() { close(release) }); <-done }) go func() { defer close(done) - changed <- f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID) + changed <- f.daemon.refreshNodeModels(original) }() select { case <-closed: @@ -1109,7 +1112,7 @@ func TestDiscovery_DiscardsQueuedModelsForChangedAddresses(t *testing.T) { var output bytes.Buffer f.daemon.codec = NewCodec(&output) - assert.False(t, f.daemon.refreshNodeModelsCandidates(peerUUID, original.IP, original.CandidateIPs(), port, peerUUID)) + assert.False(t, f.daemon.refreshNodeModels(original)) after, present := f.daemon.dir.get(peerUUID) require.True(t, present) @@ -1117,6 +1120,42 @@ func TestDiscovery_DiscardsQueuedModelsForChangedAddresses(t *testing.T) { assert.Empty(t, output.Bytes()) } +func TestDiscovery_DiscardsQueuedModelsAfterReadmission(t *testing.T) { + const peerUUID = "peer-uuid" + f := newDiscoveryPeerFixture(t, peerUUID) + f.info.Peer.Services[noderec.ServiceEngineManager] = portFromURL(t, f.server.URL) + f.respond = func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/models" { + return false + } + assert.NoError(t, json.NewEncoder(w).Encode(modelInventoryResponse{Models: []string{"reported-model"}})) + return true + } + f.discover(t) + target, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + + // Queued work keeps its original proof even when readmission leaves the + // peer's certificate and endpoints unchanged. + clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) + f.daemon.reloadTrust() + f.discover(t) + before, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + require.NotEqual(t, target.DiscoveryRecordID, before.DiscoveryRecordID) + require.Equal(t, target.CandidateIPs(), before.CandidateIPs()) + require.Equal(t, target.Services, before.Services) + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + assert.False(t, f.daemon.refreshNodeModels(target)) + after, present := f.daemon.dir.get(peerUUID) + require.True(t, present) + assert.Equal(t, before, after) + assert.Empty(t, output.Bytes()) + assert.True(t, f.daemon.refreshNodeModels(before)) +} + func TestDiscovery_PeerActivityDoesNotKeepSpoofedDiscoveryReachable(t *testing.T) { f := newDiscoveryPeerFixture(t, "peer-uuid") f.discover(t) @@ -1154,6 +1193,33 @@ func TestPeerReachable_RejectsCachedLivenessAfterReadmission(t *testing.T) { assert.True(t, f.daemon.reachable(f.raw)) } +func TestDiscovery_RequiresReauthenticationBeforeReadmittedPeerTelemetry(t *testing.T) { + f := newDiscoveryPeerFixture(t, "peer-uuid") + f.discover(t) + before, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + acceptedAt := f.daemon.lastInfoAt["peer-uuid"] + var output bytes.Buffer + f.daemon.codec = NewCodec(&output) + + // A fresh read under a new admission must not renew a directory record + // authenticated under the previous admission, even with the same certificate. + clustertrusttest.WriteAdmission(t, f.selfDir, "test-cluster", 2) + assert.False(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), before.HostUUID, before.CandidateIPs(), before.Services[noderec.ServiceNodeInfo].Port)) + after, present := f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.Equal(t, before, after) + assert.Equal(t, acceptedAt, f.daemon.lastInfoAt["peer-uuid"]) + assert.Empty(t, output.Bytes()) + + f.daemon.reloadTrust() + f.daemon.refreshCandidatesOnce(context.Background()) + after, present = f.daemon.dir.get("peer-uuid") + require.True(t, present) + assert.NotEqual(t, before.DiscoveryRecordID, after.DiscoveryRecordID) + assert.True(t, f.daemon.refreshNodeTelemetryCandidates(context.Background(), after.HostUUID, after.CandidateIPs(), after.Services[noderec.ServiceNodeInfo].Port)) +} + func TestDiscovery_RefreshesOnlyAuthenticatedInventory(t *testing.T) { const updatedPort = 11435 f := newDiscoveryPeerFixture(t, "peer-uuid") @@ -1444,7 +1510,6 @@ func TestDiscovery_RemovesExpiredDiscoveryAfterUUIDClaimChanges(t *testing.T) { assert.False(t, present, "expired unverified discovery must be removed") _, present = f.daemon.dir.get("peer-uuid") assert.False(t, present, "expired unavailable peer must be removed") - assert.Empty(t, f.daemon.peerCertificates) after, present := f.daemon.dir.get(selfUUID) require.True(t, present) assert.Equal(t, self, after, "discovery removal must preserve the local record") @@ -1494,7 +1559,6 @@ func TestDiscovery_RetainedPeerExpiresAfterAvailabilityEvidenceEnds(t *testing.T refresh() _, present = f.daemon.dir.get("peer-uuid") assert.False(t, present) - assert.Empty(t, f.daemon.peerCertificates) assert.Empty(t, f.daemon.lastInfoAt) var message Message require.NoError(t, json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message)) diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index 0afb17045..048bcc801 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -190,7 +190,7 @@ func equalOptional[T comparable](a, b *T) bool { // applyModels atomically reconciles one node's model inventory from a completed // /v1/models fetch, guarded against a node that was removed or re-addressed // while the fetch was in flight. Both the post-authentication model sweep and -// periodic refresh use this write path through refreshNodeModelsCandidates. +// periodic refresh use this write path through refreshNodeModels. // // It confirms the node still exists AND still advertises the same em endpoint // (ip + port) the fetch targeted, so an in-flight result can never resurrect a diff --git a/services/nvpair-node-scanner/refresh_test.go b/services/nvpair-node-scanner/refresh_test.go index 621739aad..6a635f16b 100644 --- a/services/nvpair-node-scanner/refresh_test.go +++ b/services/nvpair-node-scanner/refresh_test.go @@ -88,27 +88,25 @@ func portFromURL(t *testing.T, raw string) int { // behavior uses newDiscoveryPeerFixture. func newRefreshTestDaemon() *daemon { return &daemon{ - reg: newRegistry("self-node-uuid", "", selfAddrs("127.0.0.1")), - dir: newDirectory(), - modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, - lastInfoAt: make(map[string]time.Time), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), + reg: newRegistry("self-node-uuid", "", selfAddrs("127.0.0.1")), + dir: newDirectory(), + modelsHTTP: &http.Client{Timeout: modelsFetchTimeout}, + lastInfoAt: make(map[string]time.Time), } } -// seedEMNode adds a directory entry advertising engine-manager (em) at ip:port, -// with no model inventory yet (the state before any enrichment). -func seedEMNode(d *daemon, hostUUID, ip string, emPort int) { - d.dir.upsert(noderec.DirectoryNode{ +// seedEMNode adds and returns an engine-manager refresh target with no inventory. +func seedEMNode(d *daemon, hostUUID, ip string, emPort int) noderec.DirectoryNode { + target := noderec.DirectoryNode{ HostUUID: hostUUID, Name: hostUUID, IP: ip, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ noderec.ServiceEngineManager: {Port: emPort}, }, - }) + } + d.dir.upsert(target) + return target } // Model inventory converges without requiring a changed discovery announcement. @@ -118,9 +116,9 @@ func TestRefresh_PopulatesInitialEmptyInventory(t *testing.T) { "models": []string{"llama3:8b", "qwen:0.5b"}, "modelsByEngine": map[string][]string{"ollama": {"llama3:8b"}, "lmstudio": {"qwen:0.5b"}}, }) - seedEMNode(d, "peer-A", "127.0.0.1", port) + target := seedEMNode(d, "peer-A", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-A", "127.0.0.1", port, ""), "initial empty inventory should report a change once populated") + require.True(t, d.refreshNodeModels(target), "initial empty inventory should report a change once populated") got, _ := d.dir.get("peer-A") assert.Equal(t, []string{"llama3:8b", "qwen:0.5b"}, got.Models, "Models") assert.Equal(t, map[string][]string{"ollama": {"llama3:8b"}, "lmstudio": {"qwen:0.5b"}}, got.ModelsByEngine, "ModelsByEngine") @@ -132,7 +130,7 @@ func TestRefreshModels_FallsBackToASecondPublishedAddress(t *testing.T) { "models": []string{"reachable-model"}, }) const peerUUID = "peer-multihomed-models" - d.dir.upsert(noderec.DirectoryNode{ + target := noderec.DirectoryNode{ HostUUID: peerUUID, Name: peerUUID, IP: "127.0.0.2", @@ -140,9 +138,10 @@ func TestRefreshModels_FallsBackToASecondPublishedAddress(t *testing.T) { Services: map[noderec.ServiceKey]noderec.ServiceStatus{ noderec.ServiceEngineManager: {Port: port}, }, - }) + } + d.dir.upsert(target) - require.True(t, d.refreshNodeModelsCandidates(peerUUID, "127.0.0.2", []string{"127.0.0.2", "127.0.0.1"}, port, ""), "model inventory did not refresh through the second published address") + require.True(t, d.refreshNodeModels(target), "model inventory did not refresh through the second published address") got, _ := d.dir.get(peerUUID) require.Equal(t, []string{"reachable-model"}, got.Models, "Models") } @@ -158,43 +157,17 @@ func TestRefreshSelf_DialsLoopback(t *testing.T) { // TEST-NET-1 (RFC 5737): guaranteed non-routable — only a loopback dial can // reach the stub, which httptest binds on 127.0.0.1:port. const unreachableLAN = "192.0.2.1" - seedEMNode(d, "self-node-uuid", unreachableLAN, port) + target := seedEMNode(d, "self-node-uuid", unreachableLAN, port) - require.True(t, d.refreshNodeModels("self-node-uuid", unreachableLAN, port, ""), "self model refresh should converge via loopback despite an unreachable advertised LAN address") + require.True(t, d.refreshNodeModels(target), "self model refresh should converge via loopback despite an unreachable advertised LAN address") got, _ := d.dir.get("self-node-uuid") assert.Equal(t, []string{"llama3:8b"}, got.Models, "self Models") assert.Equal(t, unreachableLAN, got.IP, "self IP (%v)", unreachableLAN) } -// Republishing local services during an outage must retain the latest periodic inventory. -func TestPublishSelf_RetainsLatestModelsAfterLocalInventoryFailure(t *testing.T) { - d := newSelfTestDaemon("self-uuid", "192.0.2.1") - stub, port := startModelsStub(t, http.StatusOK, map[string]any{ - "models": []string{"initial-model"}, - "modelsByEngine": map[string][]string{"ollama": {"initial-model"}}, - "loadedByEngine": map[string][]string{"ollama": {"initial-model"}}, - }) - d.reg.register(noderec.RegisterParams{Service: noderec.ServiceEngineManager, Port: port}) - d.publishSelf() - stub.set(http.StatusOK, map[string]any{ - "models": []string{"latest-model"}, - "modelsByEngine": map[string][]string{"ollama": {"latest-model"}}, - "loadedByEngine": map[string][]string{"ollama": {"latest-model"}}, - }) - require.True(t, d.refreshNodeModels("self-uuid", "192.0.2.1", port, "")) - - stub.set(http.StatusServiceUnavailable, nil) - d.publishSelf() - - node, present := d.dir.get("self-uuid") - require.True(t, present) - assert.Equal(t, []string{"latest-model"}, node.Models) - assert.Equal(t, map[string][]string{"ollama": {"latest-model"}}, node.ModelsByEngine) - assert.Equal(t, map[string][]string{"ollama": {"latest-model"}}, node.LoadedByEngine) -} - // Publishing local services must not wait for an engine-manager read before -// owned hardware notifications and subsequent IPC can proceed. +// owned hardware notifications and subsequent IPC can proceed. The publication +// must retain flat, per-engine, and loaded model inventories together. func TestPublishSelf_UsesCachedModelsWithoutWaitingForHTTP(t *testing.T) { d := newSelfTestDaemon("self-uuid", "192.0.2.1") blockReads := make(chan struct{}) @@ -209,13 +182,19 @@ func TestPublishSelf_UsesCachedModelsWithoutWaitingForHTTP(t *testing.T) { <-release default: } - assert.NoError(t, json.NewEncoder(w).Encode(map[string][]string{"models": {"retained-model"}})) + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "models": []string{"retained-model"}, + "modelsByEngine": map[string][]string{"ollama": {"retained-model"}}, + "loadedByEngine": map[string][]string{"ollama": {"retained-model"}, "lmstudio": {}}, + })) })) t.Cleanup(server.Close) port := portFromURL(t, server.URL) d.reg.register(noderec.RegisterParams{Service: noderec.ServiceEngineManager, Port: port}) d.publishSelf() - require.True(t, d.refreshNodeModels("self-uuid", "192.0.2.1", port, "")) + target, present := d.dir.get("self-uuid") + require.True(t, present) + require.True(t, d.refreshNodeModels(target)) select { case <-d.modelsWake: default: @@ -241,6 +220,8 @@ func TestPublishSelf_UsesCachedModelsWithoutWaitingForHTTP(t *testing.T) { node, present := d.dir.get("self-uuid") require.True(t, present) assert.Equal(t, []string{"retained-model"}, node.Models) + assert.Equal(t, map[string][]string{"ollama": {"retained-model"}}, node.ModelsByEngine) + assert.Equal(t, map[string][]string{"ollama": {"retained-model"}, "lmstudio": {}}, node.LoadedByEngine) select { case <-d.modelsWake: default: @@ -255,28 +236,28 @@ func TestRefreshModels_ReportsDownloadsAndDeletionsButNotUnchangedInventory(t *t "models": []string{"a"}, "modelsByEngine": map[string][]string{"ollama": {"a"}}, }) - seedEMNode(d, "peer-B", "127.0.0.1", port) + target := seedEMNode(d, "peer-B", "127.0.0.1", port) // Initial populate is a change. - require.True(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "initial populate should report a change") + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") // Unchanged inventory: no further change events. - assert.False(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "unchanged inventory should not report a change") + assert.False(t, d.refreshNodeModels(target), "unchanged inventory should not report a change") // Pull: add a model -> exactly one change. stub.set(http.StatusOK, map[string]any{ "models": []string{"a", "b"}, "modelsByEngine": map[string][]string{"ollama": {"a", "b"}}, }) - require.True(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "a pull should report a change") - assert.False(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "re-reading the same post-pull inventory should not report a change") + require.True(t, d.refreshNodeModels(target), "a pull should report a change") + assert.False(t, d.refreshNodeModels(target), "re-reading the same post-pull inventory should not report a change") // Delete: remove a model -> exactly one change. stub.set(http.StatusOK, map[string]any{ "models": []string{"a"}, "modelsByEngine": map[string][]string{"ollama": {"a"}}, }) - require.True(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "a delete should report a change") - assert.False(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "re-reading the same post-delete inventory should not report a change") + require.True(t, d.refreshNodeModels(target), "a delete should report a change") + assert.False(t, d.refreshNodeModels(target), "re-reading the same post-delete inventory should not report a change") got, _ := d.dir.get("peer-B") assert.Equal(t, []string{"a"}, got.Models, "final Models") @@ -286,8 +267,8 @@ func TestRefreshModels_ReportsDownloadsAndDeletionsButNotUnchangedInventory(t *t "models": []string{}, "modelsByEngine": map[string][]string{"ollama": {}}, }) - require.True(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "deleting the last model should report a change") - assert.False(t, d.refreshNodeModels("peer-B", "127.0.0.1", port, ""), "re-reading the same explicit empty inventory should not report a change") + require.True(t, d.refreshNodeModels(target), "deleting the last model should report a change") + assert.False(t, d.refreshNodeModels(target), "re-reading the same explicit empty inventory should not report a change") got, _ = d.dir.get("peer-B") assert.Empty(t, got.Models, "Models after last delete") assert.Equal(t, map[string][]string{"ollama": {}}, got.ModelsByEngine, "ModelsByEngine after last delete") @@ -300,13 +281,13 @@ func TestRefreshReorderOnly_IsNotAChange(t *testing.T) { "models": []string{"a", "b"}, "modelsByEngine": map[string][]string{"ollama": {"a", "b"}}, }) - seedEMNode(d, "peer-R", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-R", "127.0.0.1", port, ""), "initial populate should report a change") + target := seedEMNode(d, "peer-R", "127.0.0.1", port) + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") stub.set(http.StatusOK, map[string]any{ "models": []string{"b", "a"}, "modelsByEngine": map[string][]string{"ollama": {"b", "a"}}, }) - assert.False(t, d.refreshNodeModels("peer-R", "127.0.0.1", port, ""), "a mere reordering of the same models should not report a change") + assert.False(t, d.refreshNodeModels(target), "a mere reordering of the same models should not report a change") } // A failed refresh preserves both flat and per-engine inventories. @@ -316,12 +297,12 @@ func TestRefreshFetchFailure_RetainsInventory(t *testing.T) { "models": []string{"keep-me"}, "modelsByEngine": map[string][]string{"ollama": {"keep-me"}}, }) - seedEMNode(d, "peer-C", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-C", "127.0.0.1", port, ""), "initial populate should report a change") + target := seedEMNode(d, "peer-C", "127.0.0.1", port) + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") // Now the endpoint fails. stub.set(http.StatusInternalServerError, nil) - assert.False(t, d.refreshNodeModels("peer-C", "127.0.0.1", port, ""), "a failed fetch should not report a change") + assert.False(t, d.refreshNodeModels(target), "a failed fetch should not report a change") got, _ := d.dir.get("peer-C") assert.Equal(t, []string{"keep-me"}, got.Models, "Models after failure") assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, got.ModelsByEngine, "ModelsByEngine after failure") @@ -334,8 +315,8 @@ func TestRefreshModels_PreservesInventoryForUnreportedEngines(t *testing.T) { "models": []string{"x", "y"}, "modelsByEngine": map[string][]string{"ollama": {"x"}}, }) - seedEMNode(d, "peer-D", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-D", "127.0.0.1", port, ""), "initial populate should report a change") + target := seedEMNode(d, "peer-D", "127.0.0.1", port) + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") got, _ := d.dir.get("peer-D") assert.Equal(t, []string{"x", "y"}, got.Models, "Models") assert.Equal(t, map[string][]string{"ollama": {"x"}}, got.ModelsByEngine, "ModelsByEngine") @@ -349,9 +330,9 @@ func TestRefresh_PopulatesLoadedByEngine(t *testing.T) { "modelsByEngine": map[string][]string{"ollama": {"llama3:8b"}, "lmstudio": {"qwen:0.5b"}}, "loadedByEngine": map[string][]string{"ollama": {"llama3:8b"}, "lmstudio": {}}, }) - seedEMNode(d, "peer-L", "127.0.0.1", port) + target := seedEMNode(d, "peer-L", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-L", "127.0.0.1", port, ""), "initial populate should report a change") + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") got, _ := d.dir.get("peer-L") assert.Equal(t, map[string][]string{"ollama": {"llama3:8b"}, "lmstudio": {}}, got.LoadedByEngine, "LoadedByEngine") } @@ -364,8 +345,8 @@ func TestRefreshLoadedOnlyChange_ReportsChange(t *testing.T) { "modelsByEngine": map[string][]string{"ollama": {"a"}}, "loadedByEngine": map[string][]string{"ollama": {}}, }) - seedEMNode(d, "peer-M", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-M", "127.0.0.1", port, ""), "initial populate should report a change") + target := seedEMNode(d, "peer-M", "127.0.0.1", port) + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") // Same installed list, but the model is now resident -> a change. stub.set(http.StatusOK, map[string]any{ @@ -373,7 +354,7 @@ func TestRefreshLoadedOnlyChange_ReportsChange(t *testing.T) { "modelsByEngine": map[string][]string{"ollama": {"a"}}, "loadedByEngine": map[string][]string{"ollama": {"a"}}, }) - require.True(t, d.refreshNodeModels("peer-M", "127.0.0.1", port, ""), "a loaded-set change should report a change even when the model list is unchanged") + require.True(t, d.refreshNodeModels(target), "a loaded-set change should report a change even when the model list is unchanged") got, _ := d.dir.get("peer-M") assert.Equal(t, map[string][]string{"ollama": {"a"}}, got.LoadedByEngine, "LoadedByEngine after load") @@ -383,7 +364,7 @@ func TestRefreshLoadedOnlyChange_ReportsChange(t *testing.T) { "modelsByEngine": map[string][]string{"ollama": {"a"}}, "loadedByEngine": map[string][]string{"ollama": {"a"}}, }) - assert.False(t, d.refreshNodeModels("peer-M", "127.0.0.1", port, ""), "an identical loaded set should not report a change") + assert.False(t, d.refreshNodeModels(target), "an identical loaded set should not report a change") } // A failed refresh must not clear the last accepted loaded-model inventory. @@ -394,11 +375,11 @@ func TestRefreshModels_RetainsLoadedModelsAfterFetchFailure(t *testing.T) { "modelsByEngine": map[string][]string{"ollama": {"keep-me"}}, "loadedByEngine": map[string][]string{"ollama": {"keep-me"}}, }) - seedEMNode(d, "peer-N", "127.0.0.1", port) - require.True(t, d.refreshNodeModels("peer-N", "127.0.0.1", port, ""), "initial populate should report a change") + target := seedEMNode(d, "peer-N", "127.0.0.1", port) + require.True(t, d.refreshNodeModels(target), "initial populate should report a change") stub.set(http.StatusInternalServerError, nil) - assert.False(t, d.refreshNodeModels("peer-N", "127.0.0.1", port, ""), "a failed fetch should not report a change") + assert.False(t, d.refreshNodeModels(target), "a failed fetch should not report a change") got, _ := d.dir.get("peer-N") assert.Equal(t, map[string][]string{"ollama": {"keep-me"}}, got.LoadedByEngine, "LoadedByEngine after failure") } @@ -406,15 +387,13 @@ func TestRefreshModels_RetainsLoadedModelsAfterFetchFailure(t *testing.T) { // A missing node must stay absent when a periodic sweep reaches its old target. func TestRefreshNodeModels_DoesNotRestoreNodeRemovedDuringSweep(t *testing.T) { d := newRefreshTestDaemon() - _, port := startModelsStub(t, http.StatusOK, map[string]any{ - "models": []string{"a"}, - }) - // The node is not in the directory (simulating removal before the in-flight - // fetch completes); the fetch succeeds but must not be applied to the - // directory. - assert.False(t, d.refreshNodeModels("gone", "127.0.0.1", port, ""), "a fetch for a removed node should report no change") + target := seedEMNode(d, "gone", "127.0.0.1", 14322) + require.True(t, d.dir.remove(target.HostUUID)) + // The sweep's target was removed before its fetch began. The directory + // lookup rejects it before any request is made. + assert.False(t, d.refreshNodeModels(target), "a refresh for a removed node should report no change") _, present := d.dir.get("gone") - assert.False(t, present, "an in-flight result must not resurrect a removed node") + assert.False(t, present, "a periodic refresh must not restore a removed node") } // Both peer handlers must be entered before either is released. This proves diff --git a/services/nvpair-tui/ui/nodesmodel_test.go b/services/nvpair-tui/ui/nodesmodel_test.go index 403c4857e..ec691e4eb 100644 --- a/services/nvpair-tui/ui/nodesmodel_test.go +++ b/services/nvpair-tui/ui/nodesmodel_test.go @@ -321,21 +321,6 @@ func TestAReusedAddress_DoesNotMergeTwoMachines(t *testing.T) { assert.Empty(t, findRow(t, rows, peerName).manualID) } -// TestTLSNodeInfoIsNotPolledInPlainText checks the detail screen shows the -// worker's reading for a TLS node rather than polling it itself. That endpoint -// needs the backend's cluster trust, and a plain-HTTP poll to it only fails. -func TestTLSNodeInfo_IsNotPolledInPlainText(t *testing.T) { - d := newNodeDetail(nil, nodeRow{ - key: "manual:m2", name: "tls-box", address: "10.0.0.9", port: 14319, - presence: presenceOnline, nodeInfoTLS: true, - probedTelemetry: &nodeTelemetry{TelemetryValid: true, - GPUs: []noderec.GPUInfo{{Name: "GPU 0", VramBytes: 1 << 30}}}, - }) - d.SetSize(100, 30) - assert.Nil(t, d.telemetryCmd(), "the detail screen must not poll a TLS node-info endpoint over plain HTTP") - assert.Contains(t, d.View(), "GPU 0", "the worker's hardware reading must be shown") -} - // Withdrawing a manual node's identity proof must remove its old hardware from // an open detail screen without starting an HTTP poll against its TLS endpoint. func TestOpenDetail_HidesManualHardwareAfterProofWithdrawal(t *testing.T) { @@ -352,7 +337,9 @@ func TestOpenDetail_HidesManualHardwareAfterProofWithdrawal(t *testing.T) { v.openDetail() require.NotNil(t, v.detail) require.Contains(t, v.View(), "GPU 0", "the authenticated hardware must be visible before proof withdrawal") + assert.Nil(t, v.detail.telemetryCmd(), "the detail screen must not poll a TLS node-info endpoint over plain HTTP") + // The engine still answers, but node-info can no longer prove its identity. manual.IdentityVerification = noderec.IdentityUnverified manual.HostUUID = "" manual.NodeInfoUp = false diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 87acdf0fc..1e9fee483 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -60,7 +60,7 @@ Unverified discoveries retain independent `discoveryRecordId` keys throughout the store and relay. They cannot overwrite a verified host, clear its telemetry, fail its workloads, or enter the scheduler's node universe. Explicit manual engine targets keep their own operational keys until authenticated. The broker -caches and forwards scanner `discovery:node-info` hardware snapshots only when +forwards scanner `discovery:node-info` hardware snapshots only when their record identity matches the current verified scanner record. ### Supervision & recovery diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index f422dadce..e063a9c69 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -639,11 +639,7 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { func(hostUUID string) { b.removeTelemetry(sourceScanner, hostUUID) }, b.updateLocalPeerMetadata, func(value noderec.NodeInfoEvent) { - cached, ok := b.store.nodeInfo(value.HostUUID, time.Now()) - if !ok { - return - } - if err := b.codec.Notify(noderec.NotifyNodeInfo, cached); err != nil { + if err := b.codec.Notify(noderec.NotifyNodeInfo, value); err != nil { slog.Warn("forward node-info failed", "host_uuid", value.HostUUID, "err", err) } }, @@ -1599,15 +1595,8 @@ func (b *Broker) clearEngineRelaySub() { b.engineRelayMu.Unlock() } -func (b *Broker) forwardManualNodesNotification(method string, params json.RawMessage) { - receivedAt := time.Now() - b.discoveryMu.Lock() - defer b.discoveryMu.Unlock() - b.handleManualNodesNotification(method, params, receivedAt) -} - -// handleManualNodesNotification runs with discoveryMu held, including when a -// worker generation check has already been made by the spawn callback. +// handleManualNodesNotification runs with discoveryMu held after the spawn +// callback has checked the worker generation. func (b *Broker) handleManualNodesNotification(method string, params json.RawMessage, receivedAt time.Time) { if b.dispatchErrorsNotif("manual-nodes", method, params) { return @@ -3643,8 +3632,8 @@ func (b *Broker) relayToSettings(msg *Message) { // nodes/list) to nvpair-manual-nodes and relays its response back. node/add // kicks off an initial probe, so the relay uses the no-timeout async // RelayRequest rather than the bounded Call. The resulting node/discovered -// notification is merged into the discovery store by -// forwardManualNodesNotification, so the snapshot stays in sync. +// notification reaches manualNotificationHandler and is merged into the +// discovery store, so the snapshot stays in sync. func (b *Broker) relayToManualNodes(msg *Message) { mn := b.getManualNodes() if mn == nil { diff --git a/services/nvpair-ui-broker/broker_test.go b/services/nvpair-ui-broker/broker_test.go index 70d34be8a..4112cd48b 100644 --- a/services/nvpair-ui-broker/broker_test.go +++ b/services/nvpair-ui-broker/broker_test.go @@ -175,11 +175,12 @@ func TestClusterTrustChange_DiscardsVerifiedRecordsAndMeasurements(t *testing.T) broker.store.Upsert(directoryToEnriched(candidate), sourceScanner) broker.relayDir.Apply(noderec.NotifyNodeDiscovered, candidate) principal := "peer" - require.True(t, broker.store.AcceptInfo(noderec.NodeInfoEvent{ + value := noderec.NodeInfoEvent{ HostUUID: "peer", ClusterUUID: "peer", DiscoveryRecordID: peer.DiscoveryRecordID, IdentityVerification: noderec.IdentityPinned, Info: noderec.NodeInfo{HostUUID: "peer", ClusterUUID: &principal}, - }, time.Now())) + } + require.True(t, broker.store.AcceptInfo(value)) clustertrusttest.RemovePeerPin(t, broker.clusterDir, "peer") broker.applyClusterTrustChange() @@ -190,8 +191,7 @@ func TestClusterTrustChange_DiscardsVerifiedRecordsAndMeasurements(t *testing.T) assert.Empty(t, broker.manualAliasForKey("peer")) assert.Equal(t, "configured", broker.manualAliasForKey("manual:configured")) assert.Equal(t, []noderec.DirectoryNode{candidate}, broker.relayDir.Snapshot("")) - _, cached := broker.store.nodeInfo("peer", time.Now()) - assert.False(t, cached) + assert.False(t, broker.store.AcceptInfo(value)) } func TestClusterTrustChange_PreservesProofsForUnchangedPins(t *testing.T) { @@ -301,7 +301,7 @@ func TestManualNodes_FailsWorkloadsWhenTrustWithdrawalPrecedesNotification(t *te // Different workers use independent streams. The manual update may // erase the peer's last record before the trust notification arrives. clustertrusttest.RemovePeerPin(t, broker.clusterDir, peerUUID) - broker.forwardManualNodesNotification("node/updated", params) + broker.manualNotificationHandler(broker.manualGeneration)("node/updated", params) require.False(t, broker.store.verifiedSource(peerUUID, sourceManual)) broker.applyClusterTrustChange() @@ -343,7 +343,7 @@ func TestManualNodes_PreservesWorkloadsAfterUnverifiedUpdateWithCurrentPin(t *te // An unsuccessful hardware probe does not revoke the peer's authorization // or establish that an inference request running on that peer has failed. - broker.forwardManualNodesNotification("node/updated", params) + broker.manualNotificationHandler(broker.manualGeneration)("node/updated", params) broker.applyClusterTrustChange() assert.False(t, broker.store.verifiedSource(peerUUID, sourceManual)) @@ -521,11 +521,12 @@ func TestManualNodes_RejectsQueuedProofAfterPinReplacement(t *testing.T) { DiscoveryRecordID: mesh.PeerRecordID("peer"), IdentityVerification: noderec.IdentityPinned, OllamaUp: true, OllamaPort: 11434, TelemetryValid: true, } + notify := b.manualNotificationHandler(b.manualGeneration) send := func() { t.Helper() params, err := json.Marshal(status) require.NoError(t, err) - b.forwardManualNodesNotification("node/updated", params) + notify("node/updated", params) } send() require.True(t, b.store.verifiedSource("peer", sourceManual)) @@ -586,7 +587,7 @@ func TestManualNodes_RecordReplacementDiscardsTelemetryBeforeDiscovery(t *testin replayed = broker.replayTelemetryToScheduler(scheduler) }) - broker.forwardManualNodesNotification("node/updated", params) + broker.manualNotificationHandler(broker.manualGeneration)("node/updated", params) require.Equal(t, len(wantReplay), replayed) codec := NewCodec(&output) diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 8dd8b998c..007bc6550 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -143,19 +143,13 @@ type storedNode struct { scanner *EnrichedNode manual *EnrichedNode lastSeen time.Time - info *nodeInfoObservation } -type nodeInfoObservation struct { - value noderec.NodeInfoEvent - receivedAt time.Time -} - -// AcceptInfo binds the complete snapshot to the scanner's current record. A -// candidate with the same claimed UUID cannot populate this cache. -func (s *discoveryStore) AcceptInfo(value noderec.NodeInfoEvent, now time.Time) bool { - s.mu.Lock() - defer s.mu.Unlock() +// AcceptInfo reports whether hardware belongs to the scanner's current verified +// record. A candidate with the same claimed UUID cannot supply its measurements. +func (s *discoveryStore) AcceptInfo(value noderec.NodeInfoEvent) bool { + s.mu.RLock() + defer s.mu.RUnlock() sn, ok := s.nodes[value.HostUUID] if !ok || sn.scanner == nil { return false @@ -175,27 +169,13 @@ func (s *discoveryStore) AcceptInfo(value noderec.NodeInfoEvent, now time.Time) } if value.Info.ClusterUUID == nil || *value.Info.ClusterUUID != value.ClusterUUID { // The hardware body must name that principal too; a missing or - // conflicting identity cannot populate the verified cache. + // conflicting identity cannot supply verified measurements. return false } } - sn.info = &nodeInfoObservation{value: value, receivedAt: now} - s.nodes[value.HostUUID] = sn return true } -func (s *discoveryStore) nodeInfo(hostUUID string, now time.Time) (noderec.NodeInfoEvent, bool) { - s.mu.RLock() - defer s.mu.RUnlock() - sn, ok := s.nodes[hostUUID] - if !ok || sn.info == nil { - return noderec.NodeInfoEvent{}, false - } - value := sn.info.value - value.Info = value.Info.Aged(now.Sub(sn.info.receivedAt)) - return value, true -} - func (s *discoveryStore) verifiedSource(hostUUID string, source nodeSource) bool { s.mu.RLock() defer s.mu.RUnlock() @@ -325,9 +305,6 @@ func (s *discoveryStore) Upsert(n EnrichedNode, source nodeSource) { cp := n s.mu.Lock() sn := s.nodes[key] - if source == sourceScanner && sn.scanner != nil && sn.scanner.DiscoveryRecordID != n.DiscoveryRecordID { - sn.info = nil - } sn.setSource(source, &cp) sn.lastSeen = time.Now() s.nodes[key] = sn @@ -358,9 +335,6 @@ func (s *discoveryStore) Remove(key string, source nodeSource) bool { return false } sn.setSource(source, nil) - if source == sourceScanner { - sn.info = nil - } gone := !sn.claimed() if gone { delete(s.nodes, key) @@ -815,11 +789,11 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { if json.Unmarshal(params, &value) != nil || s.store == nil || !s.currentRecord(value.HostUUID, value.DiscoveryRecordID, value.IdentityVerification) { return } - value.Info = value.Info.Aged(time.Since(receivedAt)) - if !s.store.AcceptInfo(value, time.Now()) { + if !s.store.AcceptInfo(value) { return } if s.onNodeInfo != nil { + value.Info = value.Info.Aged(time.Since(receivedAt)) s.onNodeInfo(value) } case "ready": diff --git a/services/nvpair-ui-broker/discovery_test.go b/services/nvpair-ui-broker/discovery_test.go index b247126db..5a78dfd57 100644 --- a/services/nvpair-ui-broker/discovery_test.go +++ b/services/nvpair-ui-broker/discovery_test.go @@ -7,7 +7,6 @@ import ( "encoding/json" "sync" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,32 +37,48 @@ func TestScannerNodeInfo_IncludesDiscoveryLockWaitInSampleAge(t *testing.T) { store := newDiscoveryStore() store.Upsert(EnrichedNode{HostUUID: "self", DiscoveryRecordID: "local:self", IdentityVerification: noderec.IdentityLocal}, sourceScanner) var stateMu sync.Mutex - scanner := &scannerProcess{store: store, stateMu: &stateMu} + var received []noderec.NodeInfoEvent + scanner := &scannerProcess{ + store: store, stateMu: &stateMu, + onNodeInfo: func(value noderec.NodeInfoEvent) { received = append(received, value) }, + } wait := holdDiscoveryNotification(t, &stateMu, func() { scanner.handleNotify(noderec.NotifyNodeInfo, json.RawMessage(`{"hostUuid":"self","discoveryRecordId":"local:self","identityVerification":"local","info":{"hostUuid":"self","telemetryValid":true,"msSince":137}}`)) }) - value, present := store.nodeInfo("self", time.Now()) - require.True(t, present) - assert.GreaterOrEqual(t, value.Info.MSSince, int64(137)+wait.Milliseconds()) + require.Len(t, received, 1) + assert.GreaterOrEqual(t, received[0].Info.MSSince, int64(137)+wait.Milliseconds()) } -func TestDiscoveryStore_RecordReplacementDiscardsHardwareSnapshot(t *testing.T) { +func TestScannerNodeInfo_RejectsSnapshotsForReplacedRecord(t *testing.T) { store := newDiscoveryStore() node := EnrichedNode{HostUUID: "peer", DiscoveryRecordID: "old-proof", IdentityVerification: noderec.IdentityPinned, Trusted: true} store.Upsert(node, sourceScanner) principal := "peer" - require.True(t, store.AcceptInfo(noderec.NodeInfoEvent{ + value := noderec.NodeInfoEvent{ HostUUID: "peer", ClusterUUID: principal, DiscoveryRecordID: "old-proof", IdentityVerification: noderec.IdentityPinned, Info: noderec.NodeInfo{HostUUID: "peer", ClusterUUID: &principal}, - }, time.Now())) + } + var received []noderec.NodeInfoEvent + scanner := &scannerProcess{store: store, onNodeInfo: func(value noderec.NodeInfoEvent) { + received = append(received, value) + }} + // Replacing a directory proof must prevent the previous worker's queued + // hardware snapshot from reaching clients under the new record. node.DiscoveryRecordID = "new-proof" store.Upsert(node, sourceScanner) + params, err := json.Marshal(value) + require.NoError(t, err) + scanner.handleNotify(noderec.NotifyNodeInfo, params) + assert.Empty(t, received) - _, cached := store.nodeInfo("peer", time.Now()) - assert.False(t, cached) + value.DiscoveryRecordID = node.DiscoveryRecordID + params, err = json.Marshal(value) + require.NoError(t, err) + scanner.handleNotify(noderec.NotifyNodeInfo, params) + assert.Equal(t, []noderec.NodeInfoEvent{value}, received) } func TestScannerProcess_RejectsQueuedEventsAfterPinReplacement(t *testing.T) { @@ -159,10 +174,9 @@ func TestScannerProcess_CandidateRemovalPreservesPeerTelemetryAndWorkloads(t *te assert.Zero(t, lost) } -// Accept hardware for the current verified record, then attempt replacement -// under a different record ID. The cache must preserve and age the accepted -// sample, invalidate it on removal, and reject a late sample after removal. -func TestDiscoveryStore_CachesSnapshotsOnlyForTheirVerifiedRecord(t *testing.T) { +// Only hardware matching the current verified record and authenticated principal +// may reach clients. Missing or conflicting identity and late samples must not. +func TestScannerNodeInfo_ForwardsSnapshotsOnlyForTheirVerifiedRecord(t *testing.T) { const hostUUID = "host-a" const peerRecordID = "peer:host-a" store := newDiscoveryStore() @@ -186,29 +200,46 @@ func TestDiscoveryStore_CachesSnapshotsOnlyForTheirVerifiedRecord(t *testing.T) GPUs: []GPUInfo{{Name: "accepted GPU"}}, }, } - now := time.Unix(1_700_000_000, 0).UTC() // arbitrary fixed instant - // The snapshot names the same host, verified record, and pinned principal - // as the scanner's current directory entry, so it may populate the cache. - require.True(t, store.AcceptInfo(value, now)) - forged := value - forged.DiscoveryRecordID = "candidate:rogue" - forged.Info.GPUs = []GPUInfo{{Name: "forged GPU"}} - // A matching host UUID and a claimed "pinned" flag cannot authorize a - // different discovery record to replace that host's accepted measurements. - assert.False(t, store.AcceptInfo(forged, now.Add(time.Second))) - accepted, ok := store.nodeInfo(hostUUID, now.Add(2*time.Second)) - // Rejecting the forged update preserves the original verified snapshot. - require.True(t, ok) - assert.Equal(t, value.Info.GPUs, accepted.Info.GPUs) - // Its original 100 ms sample age includes the two seconds spent cached. - assert.Equal(t, int64(2100), accepted.Info.MSSince) - - // Removing the verified scanner record also invalidates its cached data. - store.Remove(hostUUID, sourceScanner) - _, cached := store.nodeInfo(hostUUID, now.Add(3*time.Second)) - assert.False(t, cached) + var received []noderec.NodeInfoEvent + scanner := &scannerProcess{store: store, onNodeInfo: func(value noderec.NodeInfoEvent) { + received = append(received, value) + }} + send := func(value noderec.NodeInfoEvent) { + t.Helper() + params, err := json.Marshal(value) + require.NoError(t, err) + scanner.handleNotify(noderec.NotifyNodeInfo, params) + } + send(value) + require.Len(t, received, 1) + assert.Equal(t, value.Info.GPUs, received[0].Info.GPUs) + assert.GreaterOrEqual(t, received[0].Info.MSSince, value.Info.MSSince) + + reject := func(name string, change func(*noderec.NodeInfoEvent)) { + t.Helper() + t.Run(name, func(t *testing.T) { + forged := value + change(&forged) + params, err := json.Marshal(forged) + require.NoError(t, err) + scanner.handleNotify(noderec.NotifyNodeInfo, params) + assert.Len(t, received, 1, "unauthorized hardware must not reach clients") + }) + } + reject("different record", func(value *noderec.NodeInfoEvent) { value.DiscoveryRecordID = "candidate:rogue" }) + reject("unverified identity", func(value *noderec.NodeInfoEvent) { value.IdentityVerification = noderec.IdentityUnverified }) + reject("different hardware host", func(value *noderec.NodeInfoEvent) { value.Info.HostUUID = "other" }) + reject("different principal", func(value *noderec.NodeInfoEvent) { value.ClusterUUID = "other" }) + reject("missing hardware principal", func(value *noderec.NodeInfoEvent) { value.Info.ClusterUUID = nil }) + reject("different hardware principal", func(value *noderec.NodeInfoEvent) { + other := "other" + value.Info.ClusterUUID = &other + }) + // A late snapshot cannot recreate the missing verified directory record. - assert.False(t, store.AcceptInfo(value, now.Add(3*time.Second))) + store.Remove(hostUUID, sourceScanner) + send(value) + assert.Len(t, received, 1) } // TestDiscoveryStoreRekeysOnRename verifies that a discovered node changing diff --git a/services/nvpair-ui-broker/manualnodes_test.go b/services/nvpair-ui-broker/manualnodes_test.go index 3d2ce4fde..f7affa4dc 100644 --- a/services/nvpair-ui-broker/manualnodes_test.go +++ b/services/nvpair-ui-broker/manualnodes_test.go @@ -255,10 +255,11 @@ func TestManualProxyBridge_DoesNotBlockDiscovery(t *testing.T) { b := newManualTestBroker() b.nodeID = "self" b.setProxy(proxy) + notify := b.manualNotificationHandler(b.manualGeneration) codec := NewCodec(proxySide) added := make(chan struct{}) go func() { - b.forwardManualNodesNotification("node/discovered", json.RawMessage(`{"id":"lab","address":"192.0.2.20","ollama_up":true,"ollama_port":11434}`)) + notify("node/discovered", json.RawMessage(`{"id":"lab","address":"192.0.2.20","ollama_up":true,"ollama_port":11434}`)) close(added) }() select { @@ -280,7 +281,7 @@ func TestManualProxyBridge_DoesNotBlockDiscovery(t *testing.T) { }() removed := make(chan struct{}) go func() { - b.forwardManualNodesNotification("node/removed", json.RawMessage(`{"id":"lab"}`)) + notify("node/removed", json.RawMessage(`{"id":"lab"}`)) close(removed) }() finished := func(done <-chan struct{}, operation string) { diff --git a/services/nvpair-ui-broker/nodeinfo_test.go b/services/nvpair-ui-broker/nodeinfo_test.go index 2116b97e5..ec96f4add 100644 --- a/services/nvpair-ui-broker/nodeinfo_test.go +++ b/services/nvpair-ui-broker/nodeinfo_test.go @@ -188,54 +188,6 @@ func (w *blockedMetadataWriter) Write(data []byte) (int, error) { return w.Conn.Write(data) } -func TestPeerMetadata_UpdatesDoNotWaitForBlockedWrites(t *testing.T) { - b := &Broker{nodeID: "owned-host"} - node := noderec.DirectoryNode{ - HostUUID: "owned-host", IdentityVerification: noderec.IdentityLocal, - Name: "initial-name", - } - b.updateLocalPeerMetadata(node) - // Withhold reads until the cache update returns, simulating a worker that - // has stopped consuming its stdin. - client, server := net.Pipe() - t.Cleanup(func() { - _ = client.Close() - _ = server.Close() - }) - writer := &blockedMetadataWriter{Conn: client, started: make(chan struct{})} - b.setNodeInfo(&nodeInfoProcess{stdin: writer}) - go b.pushPeerMetadata() - select { - case <-writer.started: - case <-time.After(2 * time.Second): - require.FailNow(t, "metadata write did not start") - } - node.Name = "updated-name" - updated := make(chan struct{}) - go func() { - b.updateLocalPeerMetadata(node) - close(updated) - }() - select { - case <-updated: - case <-time.After(2 * time.Second): - require.FailNow(t, "a blocked node-info write stalled the scanner notification handler") - } - // Reading releases the blocked write; both snapshots must arrive in order. - require.NoError(t, server.SetReadDeadline(time.Now().Add(2*time.Second))) - codec := NewCodec(server) - checkName := func(want string) { - t.Helper() - msg, err := codec.Read() - require.NoError(t, err) - var params noderec.PeerMetadataParams - require.NoError(t, json.Unmarshal(msg.Params, ¶ms)) - assert.Equal(t, want, params.Peer.Name) - } - checkName("initial-name") - checkName("updated-name") -} - type heldMetadataWriter struct { frames chan []byte started chan struct{} @@ -286,25 +238,43 @@ func TestPeerMetadata_CoalescesUpdatesWhileWorkerIsBlocked(t *testing.T) { }, } b.updateLocalPeerMetadata(node) + // The first frame is already encoded when this barrier closes. Keep its + // write blocked while the registry advances to the latest pending snapshot. select { case <-writer.started: case <-time.After(2 * time.Second): require.FailNow(t, "metadata write did not start") } - for i := 1; i <= updates; i++ { - node.Name = fmt.Sprintf("updated-%d", i) + updated := make(chan struct{}) + go func() { + for i := 1; i <= updates; i++ { + node.Name = fmt.Sprintf("updated-%d", i) + b.updateLocalPeerMetadata(node) + } + node.IP = "192.0.2.21" + node.ClusterUUID = "" + delete(node.Services, noderec.ServiceOllama) b.updateLocalPeerMetadata(node) + close(updated) + }() + select { + case <-updated: + case <-time.After(2 * time.Second): + require.FailNow(t, "a blocked node-info write stalled registry updates") } - node.IP = "192.0.2.21" - node.ClusterUUID = "" - delete(node.Services, noderec.ServiceOllama) - b.updateLocalPeerMetadata(node) require.NoError(t, writer.Close()) waitForMetadataSender(t, b) require.Len(t, writer.frames, 2, "only the in-flight and latest pending snapshots should be written") - <-writer.frames // The in-flight snapshot was already encoded before updates. + // The in-flight frame stays first; all intervening updates collapse into + // the complete final frame, including removed membership and services. + var initialFrame Message + require.NoError(t, json.Unmarshal(<-writer.frames, &initialFrame)) + assert.Equal(t, noderec.MethodSetPeerMetadata, initialFrame.Method) + var initial noderec.PeerMetadataParams + require.NoError(t, json.Unmarshal(initialFrame.Params, &initial)) + assert.Equal(t, "initial", initial.Peer.Name) var frame Message require.NoError(t, json.Unmarshal(<-writer.frames, &frame)) assert.Equal(t, noderec.MethodSetPeerMetadata, frame.Method) diff --git a/services/nvpair-ui-broker/proxyaddressing_test.go b/services/nvpair-ui-broker/proxyaddressing_test.go index 13eed6a9b..dd1f7b984 100644 --- a/services/nvpair-ui-broker/proxyaddressing_test.go +++ b/services/nvpair-ui-broker/proxyaddressing_test.go @@ -318,56 +318,6 @@ func TestSubscriptions_AreTrackedPerEngine(t *testing.T) { } } -// A facade re-subscribing must drop only its own prior registration. Dropping -// every one would silence the other engines, and dropping none would double-feed -// this one. -func TestResubscribe_ReplacesOnlyThatEnginesSubscription(t *testing.T) { - client, server := net.Pipe() - t.Cleanup(func() { - _ = client.Close() - _ = server.Close() - }) - go func() { - codec := NewCodec(server) - for { - if _, err := codec.Read(); err != nil { - return - } - } - }() - - dir := relay.NewDirectory() - p := &proxyProcess{peer: NewPeer(NewCodec(client)), relayDir: dir} - - all := engines.All() - if len(all) < 2 { - t.Skip("needs at least two engines to tell the subscriptions apart") - } - first, second := all[0], all[1] - - params := func(e engines.Engine) json.RawMessage { - raw, err := json.Marshal(noderec.SubscribeParams{ - Services: []noderec.ServiceKey{e.DiscoveryService}, - }) - require.NoError(t, err, "marshal subscribe") - return raw - } - - p.handleSubscribe(first.Name, params(first)) - p.handleSubscribe(second.Name, params(second)) - - p.subMu.Lock() - firstID, secondID := p.subIDs[first.Name], p.subIDs[second.Name] - p.subMu.Unlock() - - p.handleSubscribe(first.Name, params(first)) - - p.subMu.Lock() - defer p.subMu.Unlock() - assert.NotEqual(t, firstID, p.subIDs[first.Name], "re-subscribe kept %s's original ID, so it is now double-fed", first.Name) - assert.Equal(t, secondID, p.subIDs[second.Name], "re-subscribing %s changed %s's ID", first.Name, second.Name) -} - // The read pump reports ready and wires subscriptions by matching bare method // names, so it has to see through the address. An unaddressed notification is // process-scoped and passes through untouched. @@ -537,10 +487,20 @@ func TestProxyDiscovery_KeepsEachEnginesLatestSubscriptionSnapshot(t *testing.T) require.FailNow(t, "subscription setup waited for the blocked writer") } node.Name = "latest" + var initialOllamaID, initialLMStudioID int + var currentOllamaID, currentLMStudioID int updated := make(chan struct{}) go func() { proxy.handleSubscribe("lmstudio", lmstudioParams) + // Capture the IDs after both facades have subscribed. Only Ollama's + // registration should change when it requests a different filter. + proxy.subMu.Lock() + initialOllamaID, initialLMStudioID = proxy.subIDs["ollama"], proxy.subIDs["lmstudio"] + proxy.subMu.Unlock() proxy.handleSubscribe("ollama", nodeInfoParams) + proxy.subMu.Lock() + currentOllamaID, currentLMStudioID = proxy.subIDs["ollama"], proxy.subIDs["lmstudio"] + proxy.subMu.Unlock() directory.Apply(noderec.NotifyNodeUpdated, node) close(updated) }() @@ -549,6 +509,10 @@ func TestProxyDiscovery_KeepsEachEnginesLatestSubscriptionSnapshot(t *testing.T) case <-time.After(time.Second): require.FailNow(t, "updating subscriptions waited for the blocked writer") } + // The channel close publishes the before/after IDs as well as completing + // both subscription changes while the first snapshot write is still held. + assert.NotEqual(t, initialOllamaID, currentOllamaID, "Ollama must replace its subscription") + assert.Equal(t, initialLMStudioID, currentLMStudioID, "Ollama's re-subscribe must leave LM Studio's subscription intact") codec := NewCodec(proxySide) initial, err := codec.Read() diff --git a/services/shared/clustertrusttest/clustertrusttest.go b/services/shared/clustertrusttest/clustertrusttest.go index 60cc293e9..21e098fb9 100644 --- a/services/shared/clustertrusttest/clustertrusttest.go +++ b/services/shared/clustertrusttest/clustertrusttest.go @@ -72,6 +72,13 @@ func WriteAdmission(t *testing.T, clusterDir, clusterID string, epoch uint64) { func WritePeerPin(t *testing.T, clusterDir, peerUUID string) { t.Helper() certPEM, _ := mintLeaf(t, peerUUID) + WritePeerPinFromCertificate(t, clusterDir, peerUUID, certPEM) +} + +// WritePeerPinFromCertificate pins an existing fixture's exact certificate. +// Unlike WritePeerPin, it does not mint a different leaf for the same identity. +func WritePeerPinFromCertificate(t *testing.T, clusterDir, peerUUID string, certPEM []byte) { + t.Helper() trusted := filepath.Join(clusterDir, "trusted") if err := os.MkdirAll(trusted, 0o700); err != nil { t.Fatal(err) diff --git a/services/shared/discovery/discovery.go b/services/shared/discovery/discovery.go index 9559f5ec9..439323946 100644 --- a/services/shared/discovery/discovery.go +++ b/services/shared/discovery/discovery.go @@ -20,8 +20,8 @@ // - WithMissThreshold — consecutive misses before eviction (default // missThresholdDefault). // - WithNoEviction — accumulate-only, never evict (cluster-manager). -// - WithLivenessProbe — TCP-probe a threshold-missed node before evicting it; -// if it still answers, keep it (the proxies' anti-flap guard). +// - WithLivenessProbe — check availability before evicting a missing node; +// retain it while the callback has current evidence that it is available. // - WithSelfFilter — drop our own advertisement from results (workload-manager). // - WithKeyFunc — key the node map by something other than the instance name // (cluster-manager keys by uuid= so two hosts sharing an instance name but @@ -61,58 +61,23 @@ const ( Removed EventType = "removed" ) -// probeConcurrency bounds how many liveness probes run at once. A probe against -// an address that silently drops SYNs (a firewalled or unroutable peer) burns -// its full dial timeout, so probing serially lets a few such peers push one -// reconcile past the scan interval and delay every other node's eviction or -// recovery. +// probeConcurrency bounds simultaneous availability callbacks. Checks run +// outside the browser lock so a slow callback does not block directory readers. const probeConcurrency = 8 // missThresholdDefault is how many consecutive scans a node must be absent from // before it becomes eligible for eviction. At the default 5s interval that is a // full minute of continuous silence. // -// It was 15 seconds, which is not enough. A node saturated by its own inference -// load starves its control plane of CPU: mDNS responses stop going out and -// liveness probes stop being answered, for as long as the load lasts. Fifteen -// seconds of that was read as a departure, so a busy node was dropped from the -// cluster and its in-flight work failed with it — while it was still serving -// requests. -// -// The cost of waiting longer is that a genuinely departed node lingers for -// another three quarters of a minute. That is cheap: routing skips a node that -// will not answer, so the visible effect is one request that fails and retries. -// The cost of evicting too early is losing a working node and everything running -// on it, which is not. -// -// Waiting longer does not mean looking less often. The scan interval is -// unchanged, the node-info telemetry sweep still runs every couple of seconds, -// and probeAfterMisses starts the liveness probe a quarter of the way in — so by -// the time this threshold is reached the node has been asked repeatedly and has -// declined every time, rather than having been ignored for a minute and then -// judged on one dial. -// -// This is the coarse guard. The precise one is the node's own traffic — see -// nvpair-node-scanner's activityFreshness, which keeps a node that is currently -// streaming inference regardless of what its probes say. +// The grace period avoids evicting a busy node whose multicast replies are +// delayed. Availability checks begin after probeAfterMisses; scanner uses +// recent authenticated inventory and inference activity to retain working peers. const missThresholdDefault = 12 -// probeAfterMisses is how many consecutive misses a node accumulates before the -// liveness probe starts asking about it — a quarter of the default threshold, so -// about fifteen seconds. -// -// Probing and giving up are separate decisions. The threshold above is when we -// stop believing a node is there; this is when we start checking, and it stays -// short so the minute is spent gathering evidence instead of waiting. A single -// answer resets the miss counter and the node is fully recovered, which is the -// common case for a peer whose multicast is being dropped while it is otherwise -// perfectly reachable. It also means eviction rests on a run of failed probes -// rather than the one that happens to land on the deadline. -// -// The added cost is one TCP connect per missing node per scan. It is independent -// of node-info telemetry's failure backoff because it answers a different -// question — whether an mDNS-missing node should be evicted — and it is never -// aimed at an inference port. +// probeAfterMisses is how many consecutive misses trigger an availability +// callback, about fifteen seconds at the default scan interval. A positive +// result resets the miss counter; otherwise the node continues toward eviction +// at missThresholdDefault. const probeAfterMisses = 3 // emptyScanGrace bounds how many consecutive scans that lose EVERY known node at @@ -183,10 +148,9 @@ func WithMissThreshold(n int) Option { return func(o *options) { o.missThreshold // invite flow) rather than being evicted at the miss threshold. func WithNoEviction() Option { return func(o *options) { o.noEvict = true } } -// WithLivenessProbe supplies a reachability check run against a threshold-missed +// WithLivenessProbe supplies a bounded availability check for a threshold-missed // node before it is evicted; returning true keeps the node (its miss counter is -// reset). This is the proxies' TCP-probe-before-evict guard: an mDNS miss is not -// proof a node is gone. Probes run outside the browser lock and up to +// reset). Callbacks run outside the browser lock and up to // probeConcurrency at a time, so fn must be safe for concurrent use. func WithLivenessProbe(fn func(Node) bool) Option { return func(o *options) { o.liveness = fn } } diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index 76d1611cd..b16619024 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -525,11 +525,9 @@ const ( // plus its liveness facet. // // Probe is reserved and currently always ProbeUnknown: the daemon does not yet -// report a per-service liveness facet to subscribers. The critical liveness -// guarantee — the TCP-probe-before-evict anti-flap guard — IS -// implemented on the daemon (it keeps a still-reachable node across a transient -// mDNS miss); surfacing per-service reachable/unreachable/inaccessible here is a -// deferred reporting feature (it would add a probe per service per browse). +// report per-service liveness to subscribers. Scanner retains authenticated +// peers across multicast misses using accepted inventory or recent inference +// activity; per-service status reporting remains unimplemented. type ServiceStatus struct { Port int `json:"port"` Probe ProbeStatus `json:"probe,omitempty"` diff --git a/services/tests/broker_management_test.go b/services/tests/broker_management_test.go index 5f737818b..df9a32332 100644 --- a/services/tests/broker_management_test.go +++ b/services/tests/broker_management_test.go @@ -155,7 +155,9 @@ func TestBroker_BridgesManualNodeIntoProxy(t *testing.T) { ) t.Cleanup(cleanup) - waitForMethod(t, msgs, "app:ready", 10*time.Second) + // Freshly built workers can take longer to launch on Windows. Give startup + // its own allowance before checking whether the manual route converges. + waitForMethod(t, msgs, "app:ready", 30*time.Second) const nodeName = "xproc-manual-proxy-bridge" addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":900,"method":"node/add","params":{"address":"127.0.0.1","name":%q}}`, nodeName) + "\n" diff --git a/services/tests/broker_test.go b/services/tests/broker_test.go index 291a78662..5e168ae32 100644 --- a/services/tests/broker_test.go +++ b/services/tests/broker_test.go @@ -35,6 +35,7 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/jsonrpc" + "nvpair-shared/noderec" "github.com/grandcat/zeroconf" ) @@ -42,14 +43,18 @@ import ( // availableNode mirrors the broker's external camelCase wire shape (see // AvailableNode in nvpair-ui-broker). type availableNode struct { - ID string `json:"id"` - Name string `json:"name"` - IPAddress string `json:"ipAddress"` - Port int `json:"port"` - LastSeen int64 `json:"lastSeen"` - Models []string `json:"models,omitempty"` - ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + ID string `json:"id"` + HostUUID string `json:"hostUuid"` + DiscoveryRecordID string `json:"discoveryRecordId"` + IdentityVerification noderec.IdentityVerification `json:"identityVerification"` + Trusted bool `json:"trusted"` + Name string `json:"name"` + IPAddress string `json:"ipAddress"` + Port int `json:"port"` + LastSeen int64 `json:"lastSeen"` + Models []string `json:"models,omitempty"` + ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` } type availableNodesResult struct { diff --git a/services/tests/models_http_test.go b/services/tests/models_http_test.go index 9f4fb71ad..c728fc62b 100644 --- a/services/tests/models_http_test.go +++ b/services/tests/models_http_test.go @@ -48,8 +48,8 @@ func TestDiscovery_EnrichesModelsThroughHTTP(t *testing.T) { stdin, msgs, _, cleanup := startBrokerWithDirs(t, t.TempDir(), cluster.nodeDir) t.Cleanup(cleanup) - pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 30*time.Second, func(n availableNode) bool { - return modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && + pollForNode(t, stdin, msgs, cluster.peerUUID, 30*time.Second, func(n availableNode) bool { + return n.ID == instance && modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && loadedByEngineMatch(n.LoadedByEngine) }) } diff --git a/services/tests/models_refresh_test.go b/services/tests/models_refresh_test.go index f151c3fed..c5c734a37 100644 --- a/services/tests/models_refresh_test.go +++ b/services/tests/models_refresh_test.go @@ -73,8 +73,8 @@ func TestModelsPeriodicRefresh_ConvergesWithoutMDNSChange(t *testing.T) { // Require a successful empty inventory read, not just an unverified row // whose model fields have never been populated. - pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 30*time.Second, func(n availableNode) bool { - return len(n.Models) == 0 && byEngineEqual(n.ModelsByEngine, emptyByEngine) && + pollForNode(t, stdin, msgs, cluster.peerUUID, 30*time.Second, func(n availableNode) bool { + return n.ID == instance && len(n.Models) == 0 && byEngineEqual(n.ModelsByEngine, emptyByEngine) && byEngineEqual(n.LoadedByEngine, emptyByEngine) }) t.Log("node discovered with an empty inventory; now serving models without an mDNS change") @@ -95,8 +95,8 @@ func TestModelsPeriodicRefresh_ConvergesWithoutMDNSChange(t *testing.T) { "llamacpp": {"org/router-model-GGUF:Q4_K_M"}, }, }) - pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 45*time.Second, func(n availableNode) bool { - return modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && + pollForNode(t, stdin, msgs, cluster.peerUUID, 45*time.Second, func(n availableNode) bool { + return n.ID == instance && modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && loadedByEngineMatch(n.LoadedByEngine) }) @@ -116,17 +116,18 @@ func TestModelsPeriodicRefresh_ConvergesWithoutMDNSChange(t *testing.T) { "llamacpp": {}, }, }) - pollForNode(t, stdin, msgs, instance, cluster.peerUUID, 45*time.Second, func(n availableNode) bool { - return len(n.Models) == 0 && byEngineEqual(n.ModelsByEngine, emptyByEngine) && + pollForNode(t, stdin, msgs, cluster.peerUUID, 45*time.Second, func(n availableNode) bool { + return n.ID == instance && len(n.Models) == 0 && byEngineEqual(n.ModelsByEngine, emptyByEngine) && byEngineEqual(n.LoadedByEngine, emptyByEngine) }) } -// pollForNode waits for the named, authenticated peer to satisfy pred. Matching -// its pinned identity prevents an unverified discovery from satisfying readiness. -func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, instance, hostUUID string, timeout time.Duration, pred func(availableNode) bool) { +// pollForNode waits for an authenticated peer and, when supplied, an additional +// case-specific condition. Unverified discoveries cannot satisfy readiness. +func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, hostUUID string, timeout time.Duration, pred func(availableNode) bool) { t.Helper() - deadline := time.After(timeout) + deadline := time.NewTimer(timeout) + defer deadline.Stop() ticker := time.NewTicker(1500 * time.Millisecond) defer ticker.Stop() id := 200 @@ -137,20 +138,12 @@ func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, ins require.True(t, ok, "broker stream closed unexpectedly") if msg.Method == "" && msg.ID != nil { require.Nil(t, msg.Error, "discovery:get-nodes") - var res struct { - Nodes []struct { - availableNode - HostUUID string `json:"hostUuid"` - DiscoveryRecordID string `json:"discoveryRecordId"` - IdentityVerification noderec.IdentityVerification `json:"identityVerification"` - Trusted bool `json:"trusted"` - } `json:"nodes"` - } + var res availableNodesResult require.NoError(t, json.Unmarshal(msg.Result, &res), "discovery:get-nodes") for _, node := range res.Nodes { - if node.ID == instance && node.HostUUID == hostUUID && + if node.HostUUID == hostUUID && node.IdentityVerification == noderec.IdentityPinned && node.Trusted && - node.DiscoveryRecordID != "" && pred(node.availableNode) { + node.DiscoveryRecordID != "" && (pred == nil || pred(node)) { return } } @@ -159,8 +152,8 @@ func pollForNode(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, ins case <-ticker.C: id++ sendReq(t, stdin, id, "discovery:get-nodes") - case <-deadline: - require.FailNow(t, fmt.Sprintf("timed out waiting for node %q to match predicate", instance)) + case <-deadline.C: + require.FailNow(t, fmt.Sprintf("timed out waiting for authenticated node %q to match predicate", hostUUID)) } } } diff --git a/services/tests/workload_interop_test.go b/services/tests/workload_interop_test.go index 6e6a24aa7..f55af9cd5 100644 --- a/services/tests/workload_interop_test.go +++ b/services/tests/workload_interop_test.go @@ -40,7 +40,6 @@ import ( "github.com/stretchr/testify/require" "nvpair-shared/jsonrpc" - "nvpair-shared/noderec" ) const workloadMgrService = "_nvpair-workload-manager._tcp" @@ -397,42 +396,7 @@ func TestWorkloadManager_BroadcastsLocalEventsToPeers(t *testing.T) { // mDNS candidate before a test submits work that must reach that peer. func waitForPinnedPeer(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, hostUUID string, timeout time.Duration) { t.Helper() - timer := time.NewTimer(timeout) - defer timer.Stop() - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - id := 1000 - sendReq(t, stdin, id, "discovery:get-nodes") - for { - select { - case msg, ok := <-msgs: - require.True(t, ok, "broker stream closed before peer authentication") - if msg.Method != "" || msg.ID == nil { - continue - } - require.Nil(t, msg.Error, "discovery:get-nodes") - var result struct { - Nodes []struct { - HostUUID string `json:"hostUuid"` - DiscoveryRecordID string `json:"discoveryRecordId"` - IdentityVerification noderec.IdentityVerification `json:"identityVerification"` - Trusted bool `json:"trusted"` - } `json:"nodes"` - } - require.NoError(t, json.Unmarshal(msg.Result, &result), "discovery:get-nodes") - for _, node := range result.Nodes { - if node.HostUUID == hostUUID && node.IdentityVerification == noderec.IdentityPinned && node.Trusted { - require.NotEmpty(t, node.DiscoveryRecordID) - return - } - } - case <-ticker.C: - id++ - sendReq(t, stdin, id, "discovery:get-nodes") - case <-timer.C: - require.FailNowf(t, "peer did not authenticate in discovery", "hostUuid=%s", hostUUID) - } - } + pollForNode(t, stdin, msgs, hostUUID, timeout, nil) } // postPeerEvent POSTs a peer workload frame to the manager's inter-node endpoint